use std::time::{Duration, SystemTime};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use super::actor::SessionActor;
use super::types::ClientKind;
use crate::core::config::{CacheKeepWarm, CacheTtl};
use crate::runtime::keepwarm::{
CacheAnchor, KeepWarmProbe, KeepWarmUsage, PingBill, PingFailure, PingOutcome, PingReport,
PingTap,
};
pub const LATE_GUARD: Duration = Duration::from_secs(15);
pub const BACKOFF: [Duration; 3] = [
Duration::from_secs(5),
Duration::from_secs(15),
Duration::from_secs(30),
];
pub const WALL_POLL: Duration = Duration::from_secs(15);
pub const SHARE_LIMIT_PERCENT: u64 = 10;
pub const PING_INPUT_ESTIMATE: u64 = 64;
const BOUND_EPSILON: f64 = 1e-9;
fn over_share(tokens: u64, prefix: u64) -> bool {
tokens.saturating_mul(100) > prefix.saturating_mul(SHARE_LIMIT_PERCENT)
}
pub fn governing_ttl(ttl: CacheTtl) -> Duration {
match ttl {
CacheTtl::OneHour => Duration::from_secs(3600),
CacheTtl::FiveMinutes | CacheTtl::Hybrid => Duration::from_secs(300),
}
}
pub fn safety_margin(ttl: CacheTtl) -> Duration {
match ttl {
CacheTtl::OneHour => Duration::from_secs(300),
CacheTtl::FiveMinutes | CacheTtl::Hybrid => Duration::from_secs(30),
}
}
pub fn ping_interval(ttl: CacheTtl) -> Duration {
governing_ttl(ttl) - safety_margin(ttl)
}
#[cfg(any(test, feature = "testing"))]
static INTERVAL_OVERRIDE_MS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
pub fn interval_override() -> Option<Duration> {
#[cfg(any(test, feature = "testing"))]
{
let ms = INTERVAL_OVERRIDE_MS.load(std::sync::atomic::Ordering::Relaxed);
if ms > 0 {
return Some(Duration::from_millis(ms));
}
}
None
}
#[cfg(any(test, feature = "testing"))]
pub mod testing {
use std::sync::atomic::Ordering;
use std::time::Duration;
#[must_use = "the override is cleared when the guard drops"]
pub struct IntervalOverride(());
impl IntervalOverride {
pub fn set(interval: Duration) -> Self {
let ms = u64::try_from(interval.as_millis())
.unwrap_or(u64::MAX)
.max(1);
super::INTERVAL_OVERRIDE_MS.store(ms, Ordering::Relaxed);
Self(())
}
}
impl Drop for IntervalOverride {
fn drop(&mut self) {
super::INTERVAL_OVERRIDE_MS.store(0, Ordering::Relaxed);
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct TtlEvidence {
pub downgraded: bool,
pub saw_1h: bool,
pub write_1h: u64,
pub write_5m: u64,
}
pub fn honoured_ttl(configured: CacheTtl, e: TtlEvidence) -> CacheTtl {
if configured != CacheTtl::OneHour {
return configured;
}
if e.downgraded {
return CacheTtl::FiveMinutes;
}
if e.write_1h > 0 {
return CacheTtl::OneHour;
}
if e.write_5m > 0 {
return CacheTtl::FiveMinutes;
}
if e.saw_1h {
CacheTtl::OneHour
} else {
CacheTtl::FiveMinutes
}
}
pub fn request_cost_usd(
model: &str,
ttl: CacheTtl,
read: u64,
write: u64,
input: u64,
output: u64,
) -> f64 {
match ttl {
CacheTtl::OneHour => {
crate::pricing::calculate_cost_split(model, input, output, read, 0, write)
}
CacheTtl::FiveMinutes | CacheTtl::Hybrid => {
crate::pricing::calculate_cost_split(model, input, output, read, write, 0)
}
}
}
pub fn rewrite_cost_usd(model: &str, ttl: CacheTtl, prefix_tokens: u64) -> f64 {
request_cost_usd(model, ttl, 0, prefix_tokens, 0, 0)
}
#[derive(Debug, Clone, PartialEq)]
pub enum PeriodEnd {
NothingCached,
UncachedInput {
input: u64,
prefix: u64,
},
BoundReached {
ledger_usd: f64,
bound_usd: f64,
},
PingTooExpensive {
worst_usd: f64,
bound_usd: f64,
},
HoursReached(u32),
Expired {
age_secs: u64,
},
PrefixChanged,
ContextRollover(String),
CacheMiss {
written: u64,
},
PartialMiss {
written: u64,
prefix: u64,
},
Rejected(String),
TransientExhausted(String),
CostCap {
cap: f64,
},
TurnStarted,
Compaction,
Disabled,
ParkedByRequest,
SessionEnding,
Blocked(&'static str),
}
impl std::fmt::Display for PeriodEnd {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
PeriodEnd::NothingCached => f.write_str("nothing cached"),
PeriodEnd::UncachedInput { input, prefix } => write!(
f,
"ping paid for {input} uncached input tokens (prefix {prefix})"
),
PeriodEnd::BoundReached {
ledger_usd,
bound_usd,
} => write!(
f,
"ski-rental bound reached (${ledger_usd:.4} spent of ${bound_usd:.4})"
),
PeriodEnd::PingTooExpensive {
worst_usd,
bound_usd,
} => write!(
f,
"worst-case cache-hit ping ${worst_usd:.4} exceeds the ${bound_usd:.4} rewrite it would save"
),
PeriodEnd::HoursReached(h) => write!(f, "{h}h window reached"),
PeriodEnd::Expired { age_secs } => write!(
f,
"cache expired or about to ({age_secs}s since last request; slept?)"
),
PeriodEnd::PrefixChanged => f.write_str("prefix changed"),
PeriodEnd::ContextRollover(why) => write!(f, "{why}"),
PeriodEnd::CacheMiss { written } => {
write!(f, "ping missed the cache (wrote {written} tokens)")
}
PeriodEnd::PartialMiss { written, prefix } => write!(
f,
"ping rewrote {written} of {prefix} prefix tokens (history no longer cached)"
),
PeriodEnd::Rejected(m) => write!(f, "ping rejected: {m}"),
PeriodEnd::TransientExhausted(m) => write!(f, "transient errors: {m}"),
PeriodEnd::CostCap { cap } => write!(f, "max_session_cost ${cap:.4} reached"),
PeriodEnd::TurnStarted => f.write_str("turn started"),
PeriodEnd::Compaction => f.write_str("compaction"),
PeriodEnd::Disabled => f.write_str("disabled"),
PeriodEnd::ParkedByRequest => f.write_str("parked by request"),
PeriodEnd::SessionEnding => f.write_str("session ending"),
PeriodEnd::Blocked(why) => write!(f, "{why}"),
}
}
}
#[derive(Debug, Clone)]
pub struct PeriodConfig {
pub mode: CacheKeepWarm,
pub ttl: CacheTtl,
pub model: String,
pub max_tokens: u64,
pub interval_override: Option<Duration>,
pub auto_max_hours: u32,
pub reply_tokens: Option<u64>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct EpochCarry {
pub ledger_usd: f64,
pub pings: u32,
pub prefix_tokens: u64,
pub ping_input: u64,
pub idle_since: Instant,
pub idle_wall: SystemTime,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Fire {
NotYet,
Ping,
End(PeriodEnd),
}
#[derive(Debug, Clone)]
pub struct Period {
pub id: u64,
cfg: PeriodConfig,
ttl: CacheTtl,
evidence: TtlEvidence,
idle_since: Instant,
idle_wall: SystemTime,
anchor: CacheAnchor,
fingerprint: u64,
prefix_tokens: u64,
first_write: u64,
bound_usd: f64,
ledger_usd: f64,
worst_next_usd: f64,
exposure_next_usd: f64,
ping_input: u64,
pings: u32,
retries: usize,
next_at: Instant,
next_wall: SystemTime,
in_flight: bool,
fired_age: Duration,
}
fn age(anchor: &CacheAnchor, now: Instant, wall: SystemTime) -> Duration {
let mono = now.saturating_duration_since(anchor.mono);
let wall = wall.duration_since(anchor.wall).unwrap_or_default();
mono.max(wall)
}
impl Period {
#[allow(clippy::too_many_arguments)]
pub fn start(
id: u64,
cfg: PeriodConfig,
fingerprint: u64,
last: &KeepWarmUsage,
evidence: TtlEvidence,
anchor: CacheAnchor,
now: Instant,
wall: SystemTime,
) -> Result<Period, PeriodEnd> {
Self::start_in_epoch(
id,
cfg,
fingerprint,
last,
evidence,
anchor,
now,
wall,
None,
)
}
#[allow(clippy::too_many_arguments)]
pub fn start_in_epoch(
id: u64,
cfg: PeriodConfig,
fingerprint: u64,
last: &KeepWarmUsage,
evidence: TtlEvidence,
anchor: CacheAnchor,
now: Instant,
wall: SystemTime,
carry: Option<&EpochCarry>,
) -> Result<Period, PeriodEnd> {
let cached = last.cache_read_tokens + last.cache_write_tokens;
if cached == 0 {
return Err(PeriodEnd::NothingCached);
}
let ttl = honoured_ttl(cfg.ttl, evidence);
let age_now = age(&anchor, now, wall);
if age_now + LATE_GUARD >= governing_ttl(ttl) {
return Err(PeriodEnd::Expired {
age_secs: age_now.as_secs(),
});
}
let reply = cfg
.reply_tokens
.map_or(last.output_tokens, |r| r.min(last.output_tokens));
let first_write = last.input_tokens + reply;
let prefix_tokens = cached + first_write;
let mut p = Period {
id,
cfg,
ttl,
evidence,
idle_since: now,
idle_wall: wall,
anchor,
fingerprint,
prefix_tokens,
first_write,
bound_usd: 0.0,
ledger_usd: 0.0,
worst_next_usd: 0.0,
exposure_next_usd: 0.0,
ping_input: PING_INPUT_ESTIMATE,
pings: 0,
retries: 0,
next_at: now,
next_wall: wall,
in_flight: false,
fired_age: Duration::ZERO,
};
if let Some(c) = carry {
p.ledger_usd = c.ledger_usd;
p.pings = c.pings;
p.idle_since = c.idle_since;
p.idle_wall = c.idle_wall;
if c.pings > 0 {
p.prefix_tokens = c.prefix_tokens;
p.ping_input = c.ping_input;
}
}
p.reprice();
let interval = p.interval();
p.next_at = (anchor.mono + interval).max(now);
p.next_wall = (anchor.wall + interval).max(wall);
if p.worst_next_usd > p.bound_usd * (1.0 + BOUND_EPSILON) {
return Err(PeriodEnd::PingTooExpensive {
worst_usd: p.worst_next_usd,
bound_usd: p.bound_usd,
});
}
if let Some(r) = p
.past_window(p.next_at, p.next_wall)
.or_else(|| p.past_bound())
{
return Err(r);
}
Ok(p)
}
pub fn epoch_carry(&self) -> EpochCarry {
EpochCarry {
ledger_usd: self.ledger_usd,
pings: self.pings,
prefix_tokens: self.prefix_tokens,
ping_input: self.ping_input,
idle_since: self.idle_since,
idle_wall: self.idle_wall,
}
}
fn reprice(&mut self) {
let model = self.cfg.model.as_str();
self.bound_usd = rewrite_cost_usd(model, self.ttl, self.prefix_tokens);
let write = if self.pings == 0 { self.first_write } else { 0 };
self.worst_next_usd = request_cost_usd(
model,
self.ttl,
self.prefix_tokens.saturating_sub(write),
write,
self.ping_input,
self.cfg.max_tokens,
);
let miss = |ttl| {
request_cost_usd(
model,
ttl,
0,
self.prefix_tokens,
self.ping_input,
self.cfg.max_tokens,
)
};
let mut miss_usd = miss(self.ttl);
if matches!(self.cfg.ttl, CacheTtl::OneHour | CacheTtl::Hybrid) {
miss_usd = miss_usd.max(miss(CacheTtl::OneHour));
}
self.exposure_next_usd = self.worst_next_usd.max(miss_usd);
}
fn interval(&self) -> Duration {
self.cfg
.interval_override
.unwrap_or_else(|| ping_interval(self.ttl))
}
pub fn fingerprint(&self) -> u64 {
self.fingerprint
}
pub fn pings(&self) -> u32 {
self.pings
}
pub fn ledger_usd(&self) -> f64 {
self.ledger_usd
}
pub fn bound_usd(&self) -> f64 {
self.bound_usd
}
pub fn worst_next_usd(&self) -> f64 {
self.worst_next_usd
}
pub fn exposure_next_usd(&self) -> f64 {
self.exposure_next_usd
}
pub fn prefix_tokens(&self) -> u64 {
self.prefix_tokens
}
pub fn next_at(&self) -> Instant {
self.next_at
}
pub fn mode(&self) -> CacheKeepWarm {
self.cfg.mode
}
pub fn ttl(&self) -> CacheTtl {
self.ttl
}
pub fn fired_age(&self) -> Duration {
self.fired_age
}
pub fn in_flight(&self) -> bool {
self.in_flight
}
pub fn set_mode(&mut self, mode: CacheKeepWarm) {
self.cfg.mode = mode;
}
pub fn deadline(&self, now: Instant, wall: SystemTime) -> Option<Instant> {
if self.in_flight {
return None;
}
let wall_left = self.next_wall.duration_since(wall).unwrap_or_default();
Some(self.next_at.min(now + wall_left))
}
fn past_window(&self, at: Instant, wall: SystemTime) -> Option<PeriodEnd> {
let h = match self.cfg.mode {
CacheKeepWarm::Hours(h) => h,
CacheKeepWarm::Auto if self.cfg.auto_max_hours > 0 => self.cfg.auto_max_hours,
_ => return None,
};
let window = Duration::from_secs(u64::from(h) * 3600);
let walled = wall.duration_since(self.idle_wall).unwrap_or_default();
(at.saturating_duration_since(self.idle_since) >= window || walled >= window)
.then_some(PeriodEnd::HoursReached(h))
}
fn past_bound(&self) -> Option<PeriodEnd> {
(self.ledger_usd + self.worst_next_usd > self.bound_usd * (1.0 + BOUND_EPSILON)).then_some(
PeriodEnd::BoundReached {
ledger_usd: self.ledger_usd,
bound_usd: self.bound_usd,
},
)
}
pub fn fire(
&mut self,
now: Instant,
wall: SystemTime,
session_cost: f64,
max_session_cost: Option<f64>,
) -> Fire {
if self.in_flight || (now < self.next_at && wall < self.next_wall) {
return Fire::NotYet;
}
let age_now = age(&self.anchor, now, wall);
if age_now + LATE_GUARD >= governing_ttl(self.ttl) {
return Fire::End(PeriodEnd::Expired {
age_secs: age_now.as_secs(),
});
}
if let Some(r) = self.past_window(now, wall).or_else(|| self.past_bound()) {
return Fire::End(r);
}
if let Some(cap) = max_session_cost.filter(|c| c.is_finite()) {
if session_cost + self.exposure_next_usd >= cap {
return Fire::End(PeriodEnd::CostCap { cap });
}
}
self.in_flight = true;
self.fired_age = age_now;
Fire::Ping
}
pub fn charge(&mut self, cost_usd: f64) {
self.ledger_usd += cost_usd;
}
pub fn ping_sent(
&mut self,
usage: &KeepWarmUsage,
anchor: Option<CacheAnchor>,
downgraded: bool,
) -> Result<(), PeriodEnd> {
self.in_flight = false;
self.retries = 0;
self.pings += 1;
let (read, write) = (usage.cache_read_tokens, usage.cache_write_tokens);
if read == 0 && write == 0 {
return Err(PeriodEnd::NothingCached);
}
if read == 0 {
return Err(PeriodEnd::CacheMiss { written: write });
}
if over_share(usage.input_tokens, self.prefix_tokens) {
return Err(PeriodEnd::UncachedInput {
input: usage.input_tokens,
prefix: self.prefix_tokens,
});
}
let allowed = if self.pings == 1 { self.first_write } else { 0 };
if over_share(write.saturating_sub(allowed), self.prefix_tokens) {
return Err(PeriodEnd::PartialMiss {
written: write,
prefix: self.prefix_tokens,
});
}
self.prefix_tokens = read + write;
self.ping_input = usage.input_tokens.max(1);
self.evidence.downgraded |= downgraded;
self.evidence.write_1h = usage.cache_write_1h.unwrap_or(0);
self.evidence.write_5m = usage.cache_write_5m.unwrap_or(0);
self.evidence.saw_1h |= self.evidence.write_1h > 0;
self.ttl = honoured_ttl(self.cfg.ttl, self.evidence);
self.reprice();
if let Some(a) = anchor.filter(|a| a.mono >= self.anchor.mono) {
self.anchor = a;
}
let interval = self.interval();
self.next_at = self.anchor.mono + interval;
self.next_wall = self.anchor.wall + interval;
if let Some(r) = self
.past_window(self.next_at, self.next_wall)
.or_else(|| self.past_bound())
{
return Err(r);
}
Ok(())
}
pub fn ping_failed(
&mut self,
failure: &PingFailure,
now: Instant,
wall: SystemTime,
) -> Result<Duration, PeriodEnd> {
self.in_flight = false;
let msg = match failure {
PingFailure::Fatal(m) => return Err(PeriodEnd::Rejected(m.clone())),
PingFailure::Canceled => return Err(PeriodEnd::Rejected("canceled".into())),
PingFailure::Transient(m) => m.clone(),
};
let Some(delay) = BACKOFF.get(self.retries).copied() else {
return Err(PeriodEnd::TransientExhausted(msg));
};
let age_then = age(&self.anchor, now, wall) + delay;
if age_then + LATE_GUARD >= governing_ttl(self.ttl) {
return Err(PeriodEnd::TransientExhausted(format!(
"{msg}; no retry fits before the cache expires"
)));
}
if self.ledger_usd + self.worst_next_usd > self.bound_usd * (1.0 + BOUND_EPSILON) {
return Err(PeriodEnd::BoundReached {
ledger_usd: self.ledger_usd,
bound_usd: self.bound_usd,
});
}
self.retries += 1;
self.next_at = now + delay;
self.next_wall = wall + delay;
Ok(delay)
}
}
pub(crate) struct PingTask {
pub(crate) handle: tokio::task::JoinHandle<PingReport>,
cancel: CancellationToken,
tap: PingTap,
model: String,
worst_usd: f64,
}
impl PingTask {
fn bill(&self) -> PingBill {
self.tap.bill(&self.model, self.worst_usd)
}
}
#[derive(Default)]
pub(crate) struct KeepWarmState {
configured: CacheKeepWarm,
auto_max_hours: u32,
epoch: Option<EpochCarry>,
pub(crate) override_mode: Option<CacheKeepWarm>,
pub(crate) override_slot: OverrideSlot,
pub(crate) period: Option<Period>,
pub(crate) task: Option<PingTask>,
next_period_id: u64,
turn_started_at: Option<Instant>,
last_request: Option<KeepWarmUsage>,
pub(crate) interactive_seen: bool,
last_end: Option<String>,
session_pings: u32,
}
pub type OverrideSlot = std::sync::Arc<std::sync::Mutex<Option<CacheKeepWarm>>>;
impl KeepWarmState {
pub(crate) fn new(
configured: CacheKeepWarm,
auto_max_hours: u32,
override_slot: OverrideSlot,
) -> Self {
let override_mode = *override_slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Self {
configured,
auto_max_hours,
override_mode,
override_slot,
..Self::default()
}
}
pub(crate) fn mode(&self) -> CacheKeepWarm {
self.override_mode.unwrap_or(self.configured)
}
pub(crate) fn set_override(&mut self, mode: CacheKeepWarm) {
self.override_mode = Some(mode);
*self
.override_slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(mode);
}
pub(crate) fn active(&self) -> bool {
self.period.is_some()
}
pub(crate) fn deadline(&self) -> Option<Instant> {
self.deadline_at(Instant::now(), SystemTime::now())
}
fn deadline_at(&self, now: Instant, wall: SystemTime) -> Option<Instant> {
self.period
.as_ref()
.and_then(|p| p.deadline(now, wall))
.map(|d| d.min(now + WALL_POLL))
}
}
pub(crate) async fn keepwarm_timer(deadline: Option<Instant>) {
match deadline {
Some(t) => tokio::time::sleep_until(t).await,
None => std::future::pending().await,
}
}
pub(crate) async fn poll_keepwarm_ping(
task: &mut Option<PingTask>,
) -> std::result::Result<PingReport, tokio::task::JoinError> {
match task {
Some(t) => (&mut t.handle).await,
None => std::future::pending().await,
}
}
impl SessionActor {
fn cache_keepwarm_blocker(&self) -> Option<&'static str> {
if !self.runtime.is_live() || !self.conv.is_live() || self.is_parked() {
return Some("session parked");
}
if self.streaming {
return Some("turn running");
}
if self.compact.is_some() {
return Some("compaction running");
}
if self.conv.queued_message.is_some() || !self.event_queue.is_empty() {
return Some("work queued");
}
if !self.pending_prompts.is_empty() {
return Some("prompt pending");
}
if self.driver.is_some() || self.driver_pending.is_some() {
return Some("session driver armed");
}
None
}
pub(crate) fn cache_keepwarm_note_attach(&mut self, kind: ClientKind) {
if matches!(kind, ClientKind::Tui | ClientKind::Attach) {
self.cache_keepwarm.interactive_seen = true;
}
}
pub(crate) fn cache_keepwarm_note_usage(&mut self, usage: KeepWarmUsage) {
self.cache_keepwarm.last_request = Some(usage);
self.cache_keepwarm.epoch = None;
}
pub(crate) fn cache_keepwarm_turn_started(&mut self) {
self.cache_keepwarm_end(PeriodEnd::TurnStarted);
self.cache_keepwarm.turn_started_at = Some(Instant::now());
self.cache_keepwarm.last_request = None;
}
pub(crate) fn cache_keepwarm_compaction(&mut self) {
self.cache_keepwarm_end(PeriodEnd::Compaction);
self.cache_keepwarm.turn_started_at = None;
self.cache_keepwarm.last_request = None;
self.cache_keepwarm.epoch = None;
}
pub(crate) fn cache_keepwarm_conversation_replaced(&mut self) {
self.cache_keepwarm_end(PeriodEnd::PrefixChanged);
self.cache_keepwarm.turn_started_at = None;
self.cache_keepwarm.last_request = None;
self.cache_keepwarm.epoch = None;
}
pub(crate) fn cache_keepwarm_park_requested(&mut self) {
if self.cache_keepwarm.active() && self.can_park_ignoring_keepwarm() {
self.cache_keepwarm_end(PeriodEnd::ParkedByRequest);
}
}
fn cache_keepwarm_book(&mut self, usage: &KeepWarmUsage, cost: f64) {
if (cost <= 0.0 && *usage == KeepWarmUsage::default()) || !self.conv.is_live() {
return;
}
self.conv.session_cost += cost;
self.conv.keepwarm_cost += cost;
self.conv.total_input_tokens += usage.input_tokens;
self.conv.total_output_tokens += usage.output_tokens;
self.conv.total_cache_read_tokens += usage.cache_read_tokens;
self.conv.total_cache_creation_tokens += usage.cache_write_tokens;
let t = &mut self.conv.keepwarm_tokens;
t.input += usage.input_tokens;
t.output += usage.output_tokens;
t.cache_read += usage.cache_read_tokens;
t.cache_creation += usage.cache_write_tokens;
if let Some(p) = self.cache_keepwarm.period.as_mut() {
p.charge(cost);
}
self.emit_conversation();
self.request_save();
}
fn cache_keepwarm_abort_ping(&mut self) {
let Some(task) = self.cache_keepwarm.task.take() else {
return;
};
task.cancel.cancel();
task.handle.abort();
let bill = task.bill();
if bill.cost_usd > 0.0 {
tracing::info!(
session = %self.id,
cost_usd = bill.cost_usd,
booking = ?bill.booking,
"cache keep-warm ping aborted in flight; booked"
);
}
self.cache_keepwarm_book(&bill.usage, bill.cost_usd);
}
pub(crate) fn cache_keepwarm_end(&mut self, reason: PeriodEnd) {
self.cache_keepwarm_abort_ping();
let Some(p) = self.cache_keepwarm.period.take() else {
return;
};
tracing::info!(
session = %self.id,
period = p.id,
pings = p.pings(),
ledger_usd = p.ledger_usd(),
bound_usd = p.bound_usd(),
reason = %reason,
"cache keep-warm period ended"
);
self.cache_keepwarm.last_end = Some(reason.to_string());
self.cache_keepwarm.epoch = Some(p.epoch_carry());
self.rearm_park();
}
pub(crate) async fn cache_keepwarm_on_idle(&mut self) {
let Some(turn_started_at) = self.cache_keepwarm.turn_started_at.take() else {
return;
};
self.cache_keepwarm_try_start(Some(turn_started_at)).await;
}
async fn cache_keepwarm_try_start(&mut self, turn_started_at: Option<Instant>) {
if self.cache_keepwarm.active() {
return;
}
let mode = self.cache_keepwarm.mode();
if mode.is_off() || !self.cache_keepwarm.interactive_seen {
return;
}
let Some(anchor) = self
.runtime
.cache_anchor()
.filter(|a| turn_started_at.map_or(true, |t| a.mono >= t))
else {
return;
};
let Some(last) = self.cache_keepwarm.last_request.clone() else {
return;
};
if let Some(why) = self.cache_keepwarm_blocker() {
self.cache_keepwarm.last_end = Some(format!("not started: {why}"));
return;
}
if let Some(why) = self.runtime.cache_keepwarm_ineligibility() {
self.cache_keepwarm.last_end = Some(format!("not started: {why}"));
return;
}
if !crate::runtime::keepwarm::history_ready(&self.conv.api_messages) {
return;
}
let probe: KeepWarmProbe = match self
.runtime
.cache_keepwarm_probe(&self.conv.api_messages)
.await
{
Ok(p) => p,
Err(e) => {
tracing::info!(session = %self.id, "cache keep-warm not started: {e}");
self.cache_keepwarm.last_end = Some(format!("not started: {e}"));
return;
}
};
if let Some(why) = probe.context_blocks {
let reason = PeriodEnd::ContextRollover(why);
tracing::info!(session = %self.id, reason = %reason, "cache keep-warm not started");
self.cache_keepwarm.last_end = Some(format!("not started: {reason}"));
return;
}
let cfg = PeriodConfig {
mode,
ttl: self.runtime.cache_ttl(),
model: self.runtime.model().to_string(),
max_tokens: probe.max_tokens,
interval_override: interval_override(),
auto_max_hours: self.cache_keepwarm.auto_max_hours,
reply_tokens: Some(crate::runtime::keepwarm::reply_prefix_tokens(
&self.conv.api_messages,
)),
};
let evidence = TtlEvidence {
downgraded: self.runtime.cache_ttl_downgraded(),
saw_1h: self.runtime.cache_1h_honoured(),
write_1h: last.cache_write_1h.unwrap_or(0),
write_5m: last.cache_write_5m.unwrap_or(0),
};
self.cache_keepwarm.next_period_id += 1;
let id = self.cache_keepwarm.next_period_id;
let now = Instant::now();
let carry = self.cache_keepwarm.epoch;
match Period::start_in_epoch(
id,
cfg,
probe.fingerprint,
&last,
evidence,
anchor,
now,
SystemTime::now(),
carry.as_ref(),
) {
Ok(p) => {
tracing::info!(
session = %self.id,
period = id,
mode = %mode,
ttl_secs = governing_ttl(p.ttl()).as_secs(),
first_ping_in_secs = p.next_at().saturating_duration_since(now).as_secs(),
prefix_tokens = p.prefix_tokens(),
bound_usd = p.bound_usd(),
worst_ping_usd = p.exposure_next_usd(),
"cache keep-warm period started"
);
self.cache_keepwarm.period = Some(p);
}
Err(reason) => {
tracing::info!(session = %self.id, reason = %reason, "cache keep-warm not started");
self.cache_keepwarm.last_end = Some(format!("not started: {reason}"));
}
}
}
pub(crate) async fn on_cache_keepwarm_due(&mut self) {
if self.cache_keepwarm.period.is_none() || self.cache_keepwarm.task.is_some() {
return;
}
let (now, wall) = (Instant::now(), SystemTime::now());
if self
.cache_keepwarm
.period
.as_ref()
.and_then(|p| p.deadline(now, wall))
.is_some_and(|d| d > now)
{
return;
}
if let Some(why) = self.cache_keepwarm_blocker() {
self.cache_keepwarm_end(PeriodEnd::Blocked(why));
return;
}
if !crate::runtime::keepwarm::history_ready(&self.conv.api_messages) {
self.cache_keepwarm_end(PeriodEnd::PrefixChanged);
return;
}
match self
.runtime
.cache_keepwarm_probe(&self.conv.api_messages)
.await
{
Ok(probe) => {
if let Some(why) = probe.context_blocks {
self.cache_keepwarm_end(PeriodEnd::ContextRollover(why));
return;
}
}
Err(e) => {
self.cache_keepwarm_end(PeriodEnd::Rejected(e.to_string()));
return;
}
}
let session_cost = self.conv.session_cost;
let cap = self.config.max_session_cost;
let Some(p) = self.cache_keepwarm.period.as_mut() else {
return;
};
match p.fire(Instant::now(), SystemTime::now(), session_cost, cap) {
Fire::NotYet => {}
Fire::End(reason) => self.cache_keepwarm_end(reason),
Fire::Ping => {
let expect = p.fingerprint();
let worst_usd = p.worst_next_usd();
let history = self.conv.api_messages.clone();
let mut runtime = self.runtime.clone();
runtime.share_ttl_latches(&self.runtime);
let cancel = CancellationToken::new();
let token = cancel.clone();
let tap = PingTap::default();
let task_tap = tap.clone();
let handle = tokio::spawn(async move {
runtime
.cache_keepwarm_ping(&history, expect, &token, &task_tap)
.await
});
self.cache_keepwarm.task = Some(PingTask {
handle,
cancel,
tap,
model: self.runtime.model().to_string(),
worst_usd,
});
}
}
}
pub(crate) fn on_cache_keepwarm_done(
&mut self,
result: std::result::Result<PingReport, tokio::task::JoinError>,
) {
let Some(task) = self.cache_keepwarm.task.take() else {
return;
};
let report = match result {
Ok(r) => r,
Err(e) => PingReport {
result: Err(if e.is_cancelled() {
PingFailure::Canceled
} else {
PingFailure::Fatal(format!("ping task: {e}"))
}),
usage: task.tap.usage(),
notices: Vec::new(),
},
};
let bill = task.bill();
let cost = bill.cost_usd;
self.cache_keepwarm_book(&bill.usage, cost);
for notice in &report.notices {
self.emit(crate::session::SessionEventWire::SystemNotice(
notice.clone(),
));
}
if self.cache_keepwarm.period.is_none() {
return;
}
match report.result {
Ok(PingOutcome::Sent) => {
let usage = report.usage;
self.cache_keepwarm.session_pings += 1;
let anchor = self.runtime.cache_anchor();
let downgraded = self.runtime.cache_ttl_downgraded();
let Some(p) = self.cache_keepwarm.period.as_mut() else {
return;
};
let result = p.ping_sent(&usage, anchor, downgraded);
tracing::info!(
session = %self.id,
period = p.id,
ping = p.pings(),
age_secs = p.fired_age().as_secs(),
ttl_secs = governing_ttl(p.ttl()).as_secs(),
cache_read = usage.cache_read_tokens,
cache_write = usage.cache_write_tokens,
input = usage.input_tokens,
output = usage.output_tokens,
cost_usd = cost,
ledger_usd = p.ledger_usd(),
bound_usd = p.bound_usd(),
"cache keep-warm ping"
);
if let Err(reason) = result {
if matches!(
reason,
PeriodEnd::PartialMiss { .. }
| PeriodEnd::CacheMiss { .. }
| PeriodEnd::UncachedInput { .. }
| PeriodEnd::NothingCached
) {
tracing::warn!(
session = %self.id,
reason = %reason,
"cache keep-warm ping did not refresh the cache"
);
}
self.cache_keepwarm_end(reason);
return;
}
if let Some(cap) = self.config.max_session_cost.filter(|c| c.is_finite()) {
if self.conv.session_cost >= cap {
self.cache_keepwarm_end(PeriodEnd::CostCap { cap });
}
}
}
Ok(PingOutcome::PrefixChanged { .. }) => {
self.cache_keepwarm_end(PeriodEnd::PrefixChanged);
}
Err(failure) => {
let Some(p) = self.cache_keepwarm.period.as_mut() else {
return;
};
match p.ping_failed(&failure, Instant::now(), SystemTime::now()) {
Ok(delay) => tracing::info!(
session = %self.id,
period = p.id,
retry_in_secs = delay.as_secs(),
"cache keep-warm ping failed ({failure}); retrying"
),
Err(reason) => self.cache_keepwarm_end(reason),
}
}
}
}
pub(crate) async fn cache_keepwarm_recheck(&mut self) -> Option<bool> {
let expect = self.cache_keepwarm.period.as_ref()?.fingerprint();
if self.cache_keepwarm.period.as_ref()?.in_flight() {
return Some(true);
}
let matches = crate::runtime::keepwarm::history_ready(&self.conv.api_messages)
&& self
.runtime
.cache_keepwarm_probe(&self.conv.api_messages)
.await
.is_ok_and(|p| p.fingerprint == expect);
if !matches {
self.cache_keepwarm_end(PeriodEnd::PrefixChanged);
}
Some(matches)
}
pub(crate) fn cache_keepwarm_status_json(
&self,
prefix_matches: Option<bool>,
) -> serde_json::Value {
let kw = &self.cache_keepwarm;
let (now, wall) = (Instant::now(), SystemTime::now());
let period = kw.period.as_ref().map(|p| {
let next = p
.deadline(now, wall)
.map(|d| d.saturating_duration_since(now).as_secs());
serde_json::json!({
"id": p.id,
"ttl_secs": governing_ttl(p.ttl()).as_secs(),
"pings": p.pings(),
"ledger_usd": p.ledger_usd(),
"bound_usd": p.bound_usd(),
"worst_ping_usd": p.exposure_next_usd(),
"prefix_tokens": p.prefix_tokens(),
"next_ping_in_secs": next.unwrap_or(0),
"in_flight": p.in_flight(),
})
});
serde_json::json!({
"mode": kw.mode().to_string(),
"active": kw.active(),
"period": period,
"prefix_matches": prefix_matches,
"session_pings": kw.session_pings,
"session_keepwarm_usd": self.conv.keepwarm_cost,
"last_end": kw.last_end,
})
}
pub(crate) async fn cache_keepwarm_command(&mut self, arg: &str) -> serde_json::Value {
let arg = arg.trim();
if !(arg.is_empty() || arg.eq_ignore_ascii_case("status")) {
let Some(mode) = CacheKeepWarm::parse(arg) else {
return serde_json::json!({
"kind": "error",
"text": format!(
"usage: /keepwarm [status|off|auto|<N>h] (N = 1–{}; unknown: {arg})",
crate::core::config::MAX_CACHE_KEEPWARM_HOURS
),
});
};
self.cache_keepwarm.set_override(mode);
if mode.is_off() {
self.cache_keepwarm_end(PeriodEnd::Disabled);
} else if let Some(p) = self.cache_keepwarm.period.as_mut() {
p.set_mode(mode);
} else {
self.cache_keepwarm_try_start(None).await;
}
}
let prefix_matches = self.cache_keepwarm_recheck().await;
let status = self.cache_keepwarm_status_json(prefix_matches);
serde_json::json!({
"kind": "output",
"text": render_status(&status),
"keepwarm": status,
})
}
}
pub(crate) fn render_status(s: &serde_json::Value) -> String {
let mode = s["mode"].as_str().unwrap_or("off");
let spent = s["session_keepwarm_usd"].as_f64().unwrap_or(0.0);
let pings = s["session_pings"].as_u64().unwrap_or(0);
let mut out = format!("cache keep-warm: {mode} — this session: {pings} pings, ${spent:.4}");
let p = &s["period"];
if p.is_object() {
out.push_str(&format!(
"\nactive: ping {} due in {}s (TTL {}s); ${:.4} spent of ${:.4} bound; prefix {}",
p["pings"].as_u64().unwrap_or(0) + 1,
p["next_ping_in_secs"].as_u64().unwrap_or(0),
p["ttl_secs"].as_u64().unwrap_or(0),
p["ledger_usd"].as_f64().unwrap_or(0.0),
p["bound_usd"].as_f64().unwrap_or(0.0),
match s["prefix_matches"].as_bool() {
Some(true) => "unchanged",
Some(false) => "changed",
None => "unchecked",
},
));
} else {
out.push_str("\nidle period: none");
}
if let Some(last) = s["last_end"].as_str() {
out.push_str(&format!("\nlast: {last}"));
}
out
}
#[cfg(test)]
mod tests {
use super::*;
const MIN: Duration = Duration::from_secs(60);
const OPUS: &str = "claude-opus-4-7";
const SONNET: &str = "claude-sonnet-4-6";
const P: u64 = 300_000;
fn cfg(mode: CacheKeepWarm, ttl: CacheTtl, model: &str) -> PeriodConfig {
cfg_max(mode, ttl, model, 1)
}
fn cfg_max(mode: CacheKeepWarm, ttl: CacheTtl, model: &str, max_tokens: u64) -> PeriodConfig {
PeriodConfig {
mode,
ttl,
model: model.to_string(),
max_tokens,
interval_override: None,
auto_max_hours: 0,
reply_tokens: None,
}
}
fn ev_1h() -> TtlEvidence {
TtlEvidence {
saw_1h: true,
..TtlEvidence::default()
}
}
fn anchor_at(mono: Instant, wall: SystemTime) -> CacheAnchor {
CacheAnchor { mono, wall }
}
fn usage(model: &str, read: u64, write: u64, input: u64, out: u64) -> KeepWarmUsage {
KeepWarmUsage {
input_tokens: input,
output_tokens: out,
cache_read_tokens: read,
cache_write_tokens: write,
cache_write_5m: None,
cache_write_1h: None,
model: Some(model.to_string()),
}
}
fn last_read(model: &str) -> KeepWarmUsage {
usage(model, P, 0, 0, 0)
}
fn start(c: PeriodConfig, last: &KeepWarmUsage, t0: Instant, w0: SystemTime) -> Period {
Period::start(1, c, 7, last, ev_1h(), anchor_at(t0, w0), t0, w0).expect("starts")
}
fn run_from(
cfg: PeriodConfig,
last: &KeepWarmUsage,
usages: &[KeepWarmUsage],
) -> (u32, f64, f64, PeriodEnd, Duration) {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
let model = cfg.model.clone();
let mut p = start(cfg, last, t0, w0);
let mut i = 0;
loop {
let at = p.next_at();
let wall = w0 + at.saturating_duration_since(t0);
match p.fire(at, wall, 0.0, None) {
Fire::Ping => {}
Fire::End(r) => {
return (p.pings(), p.ledger_usd(), p.bound_usd(), r, at - t0);
}
Fire::NotYet => panic!("due but not fired"),
}
let u = &usages[i.min(usages.len() - 1)];
i += 1;
p.charge(u.cost_usd(&model));
if let Err(r) = p.ping_sent(u, Some(anchor_at(at, wall)), false) {
return (p.pings(), p.ledger_usd(), p.bound_usd(), r, at - t0);
}
assert!(i < 1000, "runaway");
}
}
fn run(cfg: PeriodConfig, usages: &[KeepWarmUsage]) -> (u32, f64, f64, PeriodEnd, Duration) {
let last = last_read(&cfg.model);
run_from(cfg, &last, usages)
}
#[test]
fn review_205_possible_cache_miss_must_fit_admission_cap() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
let mut p = start(
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
&last_read(OPUS),
t0,
w0,
);
let at = p.next_at();
let wall = w0 + at.saturating_duration_since(t0);
let cap = 0.30;
let decision = p.fire(at, wall, 0.0, Some(cap));
let miss = KeepWarmUsage {
cache_write_tokens: P,
cache_write_1h: Some(P),
..KeepWarmUsage::default()
};
println!(
"REVIEW estimated_worst={} possible_write={} cap={cap} decision={decision:?}",
p.worst_next_usd,
miss.cost_usd(OPUS)
);
if matches!(decision, Fire::Ping) {
assert!(
miss.cost_usd(OPUS) <= cap,
"admitted request can exceed its session cost cap on a cache miss"
);
}
}
#[test]
fn a_cap_with_room_for_a_cache_miss_still_pings() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
let mut p = start(
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
&last_read(OPUS),
t0,
w0,
);
let exposure = p.exposure_next_usd();
let miss = usage(OPUS, 0, P, PING_INPUT_ESTIMATE, 1);
let miss = KeepWarmUsage {
cache_write_1h: Some(P),
..miss
};
assert!((exposure - miss.cost_usd(OPUS)).abs() < 1e-9, "{exposure}");
assert!(p.worst_next_usd() < p.bound_usd(), "bound basis is the read");
let at = p.next_at();
let wall = w0 + at.saturating_duration_since(t0);
assert_eq!(
p.fire(at, wall, 0.0, Some(3.0)),
Fire::End(PeriodEnd::CostCap { cap: 3.0 })
);
assert_eq!(p.fire(at, wall, 0.0, Some(3.01)), Fire::Ping);
}
#[test]
fn ski_rental_1h_is_nineteen_pure_reads_over_about_17h() {
let (pings, ledger, bound, reason, elapsed) = run(
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
&[usage(OPUS, P, 0, 0, 0)],
);
assert!((bound - 3.0).abs() < 1e-9, "{bound}");
assert_eq!(pings, 19);
assert!(ledger <= bound * (1.0 + 1e-9));
assert!(matches!(reason, PeriodEnd::BoundReached { .. }), "{reason}");
assert_eq!(elapsed, 19 * 55 * MIN);
}
#[test]
fn ski_rental_5m_is_twelve_pure_reads_over_about_55_min() {
let (pings, ledger, bound, reason, elapsed) = run(
cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS),
&[usage(OPUS, P, 0, 0, 0)],
);
assert!((bound - 1.875).abs() < 1e-9, "{bound}");
assert_eq!(pings, 12);
assert!((ledger - 1.8).abs() < 1e-9);
assert!(matches!(reason, PeriodEnd::BoundReached { .. }), "{reason}");
assert_eq!(elapsed, 12 * Duration::from_secs(270));
}
#[test]
fn measured_overhead_and_first_write_are_in_the_ledger() {
let last = usage(OPUS, P - 2_000, 0, 0, 2_000);
let first = usage(OPUS, P - 2_000, 2_000, 40, 1);
let steady = usage(OPUS, P, 0, 40, 1);
let (pings, ledger, bound, reason, _) = run_from(
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
&last,
&[first.clone(), steady.clone()],
);
assert!(matches!(reason, PeriodEnd::BoundReached { .. }), "{reason}");
let expected_ledger = first.cost_usd(OPUS) + f64::from(pings - 1) * steady.cost_usd(OPUS);
assert!(
(ledger - expected_ledger).abs() < 1e-9,
"ledger is the measured sum"
);
assert!(ledger <= bound * (1.0 + 1e-9), "never past the bound");
assert!(ledger + steady.cost_usd(OPUS) > bound);
assert_eq!(pings, 19);
}
#[test]
fn an_expensive_ping_stops_the_period_early() {
let (heavy, ..) = run(
cfg_max(CacheKeepWarm::Auto, CacheTtl::OneHour, SONNET, 5_001),
&[usage(SONNET, P, 0, 40, 5_000)],
);
let (light, ..) = run(
cfg_max(CacheKeepWarm::Auto, CacheTtl::OneHour, SONNET, 31),
&[usage(SONNET, P, 0, 40, 30)],
);
assert_eq!(heavy, 10, "rewrite $1.80 / ($0.09 read + $0.075 thinking)");
assert_eq!(light, 19);
}
#[test]
fn budgeted_thinking_output_is_accounted() {
let u = usage(SONNET, P, 0, 40, 300);
let read_only = usage(SONNET, P, 0, 0, 0);
let thinking_usd = u.cost_usd(SONNET) - read_only.cost_usd(SONNET);
assert!((thinking_usd - (300.0 * 15.0 + 40.0 * 3.0) / 1e6).abs() < 1e-12);
let (pings, ledger, ..) = run(
cfg_max(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, SONNET, 301),
std::slice::from_ref(&u),
);
assert!((ledger - f64::from(pings) * u.cost_usd(SONNET)).abs() < 1e-9);
}
#[test]
fn budgeted_thinking_worst_case_must_fit_the_bound() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
let small = usage(SONNET, 20_000, 0, 0, 0);
let err = Period::start(
1,
cfg_max(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, SONNET, 32_769),
7,
&small,
TtlEvidence::default(),
anchor_at(t0, w0),
t0,
w0,
)
.unwrap_err();
match err {
PeriodEnd::PingTooExpensive {
worst_usd,
bound_usd,
} => {
assert!((bound_usd - 0.075).abs() < 1e-9, "{bound_usd}");
assert!(worst_usd > 0.49, "{worst_usd}");
}
other => panic!("{other}"),
}
let p = start(
cfg_max(CacheKeepWarm::Auto, CacheTtl::OneHour, SONNET, 32_769),
&usage(SONNET, P, 0, 0, 0),
t0,
w0,
);
assert!(p.worst_next_usd() > 0.58 && p.worst_next_usd() < p.bound_usd());
}
#[test]
fn max_session_cost_is_checked_against_the_worst_case() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(100_000);
let mut p = start(
cfg_max(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS, 10_001),
&last_read(OPUS),
t0,
w0,
);
let at = p.next_at();
let wat = w0 + (at - t0);
assert_eq!(
p.fire(at, wat, 4.70, Some(5.0)),
Fire::End(PeriodEnd::CostCap { cap: 5.0 })
);
assert_eq!(
p.fire(at, wat, 4.50, Some(5.0)),
Fire::End(PeriodEnd::CostCap { cap: 5.0 })
);
assert_eq!(p.fire(at, wat, 1.70, Some(5.0)), Fire::Ping);
}
#[test]
fn small_prompt_below_the_cache_minimum_never_starts() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH;
let below_minimum = usage(OPUS, 0, 0, 800, 20);
assert_eq!(
Period::start(
1,
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
7,
&below_minimum,
ev_1h(),
anchor_at(t0, w0),
t0,
w0
)
.unwrap_err(),
PeriodEnd::NothingCached
);
}
fn fired(c: PeriodConfig) -> (Period, Instant, SystemTime) {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(90_000);
let mut p = start(c, &last_read(OPUS), t0, w0);
let at = p.next_at();
let wat = w0 + (at - t0);
assert_eq!(p.fire(at, wat, 0.0, None), Fire::Ping);
(p, at, wat)
}
#[test]
fn a_ping_that_cached_nothing_ends_the_period() {
let (mut p, ..) = fired(cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS));
assert_eq!(
p.ping_sent(&usage(OPUS, 0, 0, P, 1), None, false),
Err(PeriodEnd::NothingCached)
);
}
#[test]
fn a_ping_paying_for_a_large_uncached_share_ends_the_period() {
let (mut p, ..) = fired(cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS));
assert_eq!(
p.ping_sent(&usage(OPUS, P - 60_000, 0, 60_000, 1), None, false),
Err(PeriodEnd::UncachedInput {
input: 60_000,
prefix: P
})
);
let (mut p, ..) = fired(cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS));
assert_eq!(p.ping_sent(&usage(OPUS, P, 0, 40, 1), None, false), Ok(()));
}
#[test]
fn a_ping_that_writes_instead_of_reading_ends_the_period() {
let (mut p, ..) = fired(cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS));
assert_eq!(
p.ping_sent(&usage(OPUS, 0, P, 40, 1), None, false),
Err(PeriodEnd::CacheMiss { written: P })
);
}
#[test]
fn a_partial_miss_after_ping_one_ends_the_period() {
let (mut p, at, wat) = fired(cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS));
p.ping_sent(&usage(OPUS, P, 0, 40, 1), Some(anchor_at(at, wat)), false)
.unwrap();
let at = p.next_at();
let wat = wat + (at - p.anchor.mono);
assert_eq!(p.fire(at, wat, 0.0, None), Fire::Ping);
assert_eq!(
p.ping_sent(&usage(OPUS, P - 50_000, 50_000, 40, 1), None, false),
Err(PeriodEnd::PartialMiss {
written: 50_000,
prefix: P
})
);
}
#[test]
fn ping_one_may_write_the_last_reply() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(3_000);
let last = usage(OPUS, 250_000, 0, 0, 50_000);
let mut p = start(
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
&last,
t0,
w0,
);
assert_eq!(p.prefix_tokens(), P);
let at = p.next_at();
assert_eq!(p.fire(at, w0 + (at - t0), 0.0, None), Fire::Ping);
assert_eq!(
p.ping_sent(&usage(OPUS, 250_000, 50_000, 40, 1), None, false),
Ok(()),
"the reply's write is expected on ping 1"
);
}
#[test]
fn honoured_ttl_needs_evidence_for_1h() {
use CacheTtl::*;
let none = TtlEvidence::default();
assert_eq!(honoured_ttl(FiveMinutes, ev_1h()), FiveMinutes);
assert_eq!(honoured_ttl(Hybrid, ev_1h()), Hybrid);
assert_eq!(honoured_ttl(OneHour, none), FiveMinutes, "no evidence");
assert_eq!(honoured_ttl(OneHour, ev_1h()), OneHour);
let wrote_1h = TtlEvidence {
write_1h: 10,
..none
};
assert_eq!(honoured_ttl(OneHour, wrote_1h), OneHour);
let wrote_5m = TtlEvidence {
saw_1h: true,
write_5m: 10,
..none
};
assert_eq!(
honoured_ttl(OneHour, wrote_5m),
FiveMinutes,
"5m-only write"
);
let downgraded = TtlEvidence {
downgraded: true,
write_1h: 10,
saw_1h: true,
..none
};
assert_eq!(honoured_ttl(OneHour, downgraded), FiveMinutes, "latch wins");
}
#[test]
fn a_silent_downgrade_governs_by_5m() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(4_000);
let downgraded = TtlEvidence {
downgraded: true,
..TtlEvidence::default()
};
let p = Period::start(
1,
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
7,
&last_read(OPUS),
downgraded,
anchor_at(t0, w0),
t0,
w0,
)
.unwrap();
assert_eq!(p.ttl(), CacheTtl::FiveMinutes);
assert_eq!(p.next_at(), t0 + Duration::from_secs(270));
assert!((p.bound_usd() - 1.875).abs() < 1e-9);
let mut p = Period::start(
2,
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
7,
&usage(OPUS, P - 1_000, 0, 0, 1_000),
TtlEvidence::default(),
anchor_at(t0, w0),
t0,
w0,
)
.unwrap();
assert_eq!(p.ttl(), CacheTtl::FiveMinutes);
let at = p.next_at();
let wat = w0 + (at - t0);
assert_eq!(p.fire(at, wat, 0.0, None), Fire::Ping);
let mut first = usage(OPUS, P - 1_000, 1_000, 40, 1);
first.cache_write_1h = Some(1_000);
first.cache_write_5m = Some(0);
p.ping_sent(&first, Some(anchor_at(at, wat)), false)
.unwrap();
assert_eq!(p.ttl(), CacheTtl::OneHour);
assert_eq!(p.next_at(), at + 55 * MIN);
let at = p.next_at();
let wat = wat + 55 * MIN;
assert_eq!(p.fire(at, wat, 0.0, None), Fire::Ping);
p.ping_sent(&usage(OPUS, P, 0, 40, 1), Some(anchor_at(at, wat)), true)
.unwrap();
assert_eq!(p.ttl(), CacheTtl::FiveMinutes);
assert_eq!(p.next_at(), at + Duration::from_secs(270));
}
#[test]
fn five_minute_pings_are_scheduled_from_the_anchor() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(5_000);
let c = cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS);
let idle = t0 + MIN;
let mut p = Period::start(
1,
c,
7,
&last_read(OPUS),
ev_1h(),
anchor_at(t0, w0),
idle,
w0 + MIN,
)
.unwrap();
assert_eq!(
p.deadline(idle, w0 + MIN),
Some(t0 + Duration::from_secs(270))
);
assert_eq!(
p.fire(
t0 + Duration::from_secs(200),
w0 + Duration::from_secs(200),
0.0,
None
),
Fire::NotYet
);
let at = t0 + Duration::from_secs(270);
let wat = w0 + Duration::from_secs(270);
assert_eq!(p.fire(at, wat, 0.0, None), Fire::Ping);
assert_eq!(p.deadline(at, wat), None, "no second ping while in flight");
assert_eq!(p.fire(at, wat, 0.0, None), Fire::NotYet);
let sent = anchor_at(at + Duration::from_secs(2), wat + Duration::from_secs(2));
p.ping_sent(&usage(OPUS, P, 0, 0, 1), Some(sent), false)
.unwrap();
assert_eq!(
p.deadline(sent.mono, sent.wall),
Some(sent.mono + Duration::from_secs(270))
);
}
#[test]
fn hybrid_is_governed_by_the_5m_history_marker() {
assert_eq!(governing_ttl(CacheTtl::Hybrid), Duration::from_secs(300));
assert_eq!(governing_ttl(CacheTtl::OneHour), Duration::from_secs(3600));
assert_eq!(ping_interval(CacheTtl::Hybrid), Duration::from_secs(270));
assert!((rewrite_cost_usd(OPUS, CacheTtl::Hybrid, P) - 1.875).abs() < 1e-9);
let (pings, ..) = run(
cfg(CacheKeepWarm::Auto, CacheTtl::Hybrid, OPUS),
&[usage(OPUS, P, 0, 0, 0)],
);
assert_eq!(pings, 12);
}
#[test]
fn a_turn_that_streamed_past_the_interval_pings_at_once_or_not_at_all() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(9_000);
let c = || cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS);
let at = |s: u64| (t0 + Duration::from_secs(s), w0 + Duration::from_secs(s));
let (idle, widle) = at(280);
let p = Period::start(
1,
c(),
7,
&last_read(OPUS),
ev_1h(),
anchor_at(t0, w0),
idle,
widle,
)
.unwrap();
assert_eq!(p.deadline(idle, widle), Some(idle), "due immediately");
let (late, wlate) = at(290);
let err = Period::start(
1,
c(),
7,
&last_read(OPUS),
ev_1h(),
anchor_at(t0, w0),
late,
wlate,
)
.unwrap_err();
assert!(matches!(err, PeriodEnd::Expired { age_secs: 290 }), "{err}");
}
#[test]
fn sleep_past_expiry_ends_the_period_without_pinging() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(50_000);
for ttl in [CacheTtl::FiveMinutes, CacheTtl::OneHour] {
let c = cfg(CacheKeepWarm::Auto, ttl, OPUS);
let mut p = start(c, &last_read(OPUS), t0, w0);
let due = p.next_at();
let wall = w0 + Duration::from_secs(3 * 3600);
match p.fire(due, wall, 0.0, None) {
Fire::End(PeriodEnd::Expired { age_secs }) => assert_eq!(age_secs, 3 * 3600),
other => panic!("{ttl:?}: {other:?}"),
}
}
}
#[test]
fn a_suspend_pulls_the_deadline_forward_by_the_wall_clock() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(60_000);
let mut p = start(
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
&last_read(OPUS),
t0,
w0,
);
let (now, wall) = (t0 + 10 * MIN, w0 + 40 * MIN);
assert_eq!(p.deadline(now, wall), Some(now + 15 * MIN));
assert_eq!(p.fire(now, wall, 0.0, None), Fire::NotYet);
let (now, wall) = (now + 15 * MIN, wall + 15 * MIN);
assert_eq!(p.fire(now, wall, 0.0, None), Fire::Ping);
assert_eq!(p.fired_age(), 55 * MIN);
}
#[test]
fn the_actor_polls_the_wall_clock_while_a_period_is_active() {
assert!(WALL_POLL < safety_margin(CacheTtl::FiveMinutes));
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(61_000);
let mut kw = KeepWarmState::default();
assert_eq!(kw.deadline_at(t0, w0), None, "no period: no timer");
kw.period = Some(start(
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
&last_read(OPUS),
t0,
w0,
));
assert_eq!(kw.deadline_at(t0, w0), Some(t0 + WALL_POLL));
let (near, wnear) = (
t0 + 55 * MIN - Duration::from_secs(3),
w0 + 55 * MIN - Duration::from_secs(3),
);
assert_eq!(kw.deadline_at(near, wnear), Some(t0 + 55 * MIN));
let p = kw.period.as_mut().unwrap();
assert_eq!(p.fire(t0 + 55 * MIN, w0 + 55 * MIN, 0.0, None), Fire::Ping);
assert_eq!(kw.deadline_at(t0 + 55 * MIN, w0 + 55 * MIN), None);
}
#[test]
fn transient_errors_back_off_within_the_ttl_then_end() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(70_000);
let mut p = start(
cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS),
&last_read(OPUS),
t0,
w0,
);
let mut now = p.next_at();
let transient = PingFailure::Transient("HTTP 529".into());
for want in BACKOFF {
let wall = w0 + now.saturating_duration_since(t0);
assert_eq!(p.fire(now, wall, 0.0, None), Fire::Ping);
assert_eq!(p.ping_failed(&transient, now, wall), Ok(want));
assert_eq!(p.deadline(now, wall), Some(now + want));
now += want;
}
let wall = w0 + now.saturating_duration_since(t0);
assert_eq!(p.fire(now, wall, 0.0, None), Fire::Ping);
assert!(matches!(
p.ping_failed(&transient, now, wall),
Err(PeriodEnd::TransientExhausted(_))
));
let mut p = start(
cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS),
&last_read(OPUS),
t0,
w0,
);
let at = p.next_at();
let wat = w0 + Duration::from_secs(270);
assert_eq!(p.fire(at, wat, 0.0, None), Fire::Ping);
assert_eq!(p.ping_failed(&transient, at, wat), Ok(BACKOFF[0]));
let at = at + BACKOFF[0];
let wat = wat + BACKOFF[0];
assert_eq!(p.fire(at, wat, 0.0, None), Fire::Ping);
assert!(matches!(
p.ping_failed(&transient, at, wat),
Err(PeriodEnd::TransientExhausted(_))
));
}
#[test]
fn failed_ping_spend_is_in_the_ledger() {
let (mut p, at, wat) = fired(cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, OPUS));
p.charge(1.80);
assert!(matches!(
p.ping_failed(&PingFailure::Transient("x".into()), at, wat),
Err(PeriodEnd::BoundReached { .. })
));
}
#[test]
fn a_4xx_ends_the_period() {
let (mut p, at, wat) = fired(cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS));
let err = p
.ping_failed(&PingFailure::Fatal("HTTP 400".into()), at, wat)
.unwrap_err();
assert_eq!(err, PeriodEnd::Rejected("HTTP 400".into()));
}
#[test]
fn hours_window_and_bound_whichever_first() {
let (pings, _, _, reason, _) = run(
cfg(CacheKeepWarm::Hours(1), CacheTtl::OneHour, OPUS),
&[usage(OPUS, P, 0, 0, 0)],
);
assert_eq!((pings, reason), (1, PeriodEnd::HoursReached(1)));
let (pings, _, _, reason, _) = run(
cfg(CacheKeepWarm::Hours(2), CacheTtl::FiveMinutes, OPUS),
&[usage(OPUS, P, 0, 0, 0)],
);
assert_eq!(pings, 12);
assert!(matches!(reason, PeriodEnd::BoundReached { .. }));
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH;
let mut c = cfg(CacheKeepWarm::Hours(1), CacheTtl::OneHour, OPUS);
c.interval_override = Some(Duration::from_secs(2 * 3600));
assert_eq!(
Period::start(
1,
c,
7,
&last_read(OPUS),
ev_1h(),
anchor_at(t0, w0),
t0,
w0
)
.unwrap_err(),
PeriodEnd::HoursReached(1)
);
}
#[test]
#[serial_test::serial(synaps_keepwarm_interval)]
fn the_interval_override_is_set_only_by_code_never_by_the_environment() {
let _env = crate::test_env::EnvVarGuard::set("SYNAPS_CACHE_KEEPWARM_INTERVAL_MS", "250");
assert_eq!(interval_override(), None, "the old env var does nothing");
{
let _g = testing::IntervalOverride::set(Duration::from_millis(250));
assert_eq!(interval_override(), Some(Duration::from_millis(250)));
}
assert_eq!(interval_override(), None, "cleared with the guard");
}
#[test]
fn a_re_enabled_period_continues_the_exhausted_ledger() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(80_000);
let c = || cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS);
let mut p = start(c(), &last_read(OPUS), t0, w0);
let read = usage(OPUS, P, 0, 0, 0);
let end = loop {
let at = p.next_at();
let wat = w0 + (at - t0);
match p.fire(at, wat, 0.0, None) {
Fire::Ping => {}
Fire::End(r) => break r,
Fire::NotYet => unreachable!(),
}
p.charge(read.cost_usd(OPUS));
if let Err(r) = p.ping_sent(&read, Some(anchor_at(at, wat)), false) {
break r;
}
};
assert!(matches!(end, PeriodEnd::BoundReached { .. }), "{end}");
let carry = p.epoch_carry();
assert_eq!(carry.pings, 19);
let anchor = p.anchor;
let err = Period::start_in_epoch(
2,
c(),
7,
&last_read(OPUS),
ev_1h(),
anchor,
anchor.mono,
anchor.wall,
Some(&carry),
)
.unwrap_err();
assert!(matches!(err, PeriodEnd::BoundReached { .. }), "{err}");
assert!(Period::start_in_epoch(
3,
c(),
7,
&last_read(OPUS),
ev_1h(),
anchor,
anchor.mono,
anchor.wall,
None
)
.is_ok());
}
#[test]
fn ping_one_write_allowance_is_never_re_granted_in_an_epoch() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(81_000);
let c = || cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS);
let last = usage(OPUS, 250_000, 0, 0, 50_000);
let mut p = start(c(), &last, t0, w0);
let at = p.next_at();
let wat = w0 + (at - t0);
assert_eq!(p.fire(at, wat, 0.0, None), Fire::Ping);
let first = usage(OPUS, 250_000, 50_000, 40, 1);
p.charge(first.cost_usd(OPUS));
p.ping_sent(&first, Some(anchor_at(at, wat)), false)
.unwrap();
let carry = p.epoch_carry();
let mut q = Period::start_in_epoch(
2,
c(),
7,
&last,
ev_1h(),
anchor_at(at, wat),
at,
wat,
Some(&carry),
)
.unwrap();
assert_eq!(q.pings(), 1);
assert!((q.ledger_usd() - first.cost_usd(OPUS)).abs() < 1e-12);
let at2 = q.next_at();
let wat2 = wat + (at2 - at);
assert_eq!(q.fire(at2, wat2, 0.0, None), Fire::Ping);
assert_eq!(
q.ping_sent(&usage(OPUS, 250_000, 50_000, 40, 1), None, false),
Err(PeriodEnd::PartialMiss {
written: 50_000,
prefix: P
})
);
}
#[test]
fn ping_counts_per_multiplier_and_the_auto_cap() {
for (model, uncapped, capped) in [
("claude-opus-4-7", 19, 19),
("claude-opus-5-5", 39, 26),
("claude-fable-5-1", 79, 26),
] {
let mut c = cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, model);
let (pings, _, _, reason, elapsed) = run(c.clone(), &[usage(model, P, 0, 0, 0)]);
assert_eq!(pings, uncapped, "{model}");
assert!(
matches!(reason, PeriodEnd::BoundReached { .. }),
"{model}: {reason}"
);
assert_eq!(elapsed, uncapped * 55 * MIN, "{model}");
c.auto_max_hours = 24;
let (pings, _, _, reason, _) = run(c, &[usage(model, P, 0, 0, 0)]);
assert_eq!(pings, capped, "{model}");
if capped < uncapped {
assert_eq!(reason, PeriodEnd::HoursReached(24), "{model}");
}
}
for (model, want) in [
("claude-opus-4-7", 12),
("claude-opus-5-5", 24),
("claude-fable-5-1", 49),
] {
let mut c = cfg(CacheKeepWarm::Auto, CacheTtl::FiveMinutes, model);
c.auto_max_hours = 24;
let (pings, ..) = run(c, &[usage(model, P, 0, 0, 0)]);
assert_eq!(pings, want, "{model} 5m");
}
let mut c = cfg(
CacheKeepWarm::Hours(48),
CacheTtl::OneHour,
"claude-fable-5-1",
);
c.auto_max_hours = 24;
let (pings, ..) = run(c, &[usage("claude-fable-5-1", P, 0, 0, 0)]);
assert_eq!(pings, 52, "48h window: 52 pings at 55 min");
}
#[test]
fn first_write_counts_only_what_the_prefix_will_hold() {
let t0 = Instant::now();
let w0 = SystemTime::UNIX_EPOCH + Duration::from_secs(82_000);
let last = usage(OPUS, 250_000, 0, 1_000, 40_000);
let mut c = cfg(CacheKeepWarm::Auto, CacheTtl::OneHour, OPUS);
let billed = start(c.clone(), &last, t0, w0);
assert_eq!(billed.prefix_tokens(), 291_000, "without the estimate");
c.reply_tokens = Some(2_000);
let p = start(c.clone(), &last, t0, w0);
assert_eq!(p.prefix_tokens(), 253_000);
c.reply_tokens = Some(90_000);
assert_eq!(start(c, &last, t0, w0).prefix_tokens(), 291_000);
}
#[test]
fn status_renders_spend_and_period() {
let s = serde_json::json!({
"mode": "auto",
"active": true,
"period": {"id": 3, "ttl_secs": 300, "pings": 2, "ledger_usd": 0.3,
"bound_usd": 1.875, "prefix_tokens": P, "next_ping_in_secs": 120,
"in_flight": false},
"prefix_matches": true,
"session_pings": 5,
"session_keepwarm_usd": 0.75,
"last_end": "prefix changed",
});
let text = render_status(&s);
assert!(text.contains("auto"), "{text}");
assert!(text.contains("5 pings, $0.7500"), "{text}");
assert!(text.contains("ping 3 due in 120s"), "{text}");
assert!(text.contains("$0.3000 spent of $1.8750 bound"), "{text}");
assert!(text.contains("prefix unchanged"), "{text}");
assert!(text.contains("last: prefix changed"), "{text}");
}
}