aisimulate_core/replay/core/
mod.rs

1// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Router-neutral contracts for offline replay.
5//!
6//! Runtime adapters and concrete policies live one level up so this directory
7//! can later become a standalone crate without deployment-specific APIs.
8
9use anyhow::Result;
10use uuid::Uuid;
11
12use crate::replay::ReplayTerminalStatus;
13
14pub mod round_robin;
15
16pub trait RequestIdentity {
17    fn request_id(&self) -> Option<Uuid>;
18
19    /// An authored attention-DP preference. Policies that do not provide
20    /// rank-affinity semantics may ignore it.
21    fn preferred_dp_rank(&self) -> Option<u32> {
22        None
23    }
24}
25
26#[derive(Debug)]
27pub struct ReadyArrival<Request, Metadata> {
28    pub request: Request,
29    pub arrival_time_ms: f64,
30    pub metadata: Metadata,
31    pub authored_request_id: Option<String>,
32    pub play_id: Option<String>,
33    pub dispatched_at_ms: f64,
34    pub session_id: Option<String>,
35    pub turn_index: Option<usize>,
36}
37
38pub trait AdmissionSource {
39    type Request;
40    type Metadata;
41
42    fn next_ready_time_ms(&mut self) -> Option<f64>;
43    fn drain_ready(
44        &mut self,
45        now_ms: f64,
46        cluster_in_flight: usize,
47    ) -> Result<Vec<ReadyArrival<Self::Request, Self::Metadata>>>;
48    fn on_output_token(&mut self, request_id: Uuid, token_id: u32) -> Result<()>;
49    fn on_terminal(
50        &mut self,
51        request_id: Uuid,
52        now_ms: f64,
53        status: ReplayTerminalStatus,
54    ) -> Result<()>;
55    fn is_drained(&self) -> bool;
56    fn total_requests(&self) -> usize;
57}
58
59#[derive(Debug, Clone, Copy, PartialEq, Eq)]
60pub struct PlacementCacheSample {
61    /// Prefix blocks available on the selected worker.
62    pub overlap_blocks: u32,
63    /// Largest prefix overlap available on any eligible worker at selection time.
64    pub best_available_overlap_blocks: u32,
65    pub isl_blocks: u32,
66}
67
68#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69pub struct Placement {
70    pub request_id: Uuid,
71    pub scheduler_id: usize,
72    pub reported_overlap_tokens: usize,
73    pub cache_sample: Option<PlacementCacheSample>,
74    /// Placement-policy replica that made the decision, when the policy has replicas.
75    pub placement_replica_id: Option<usize>,
76}
77
78#[derive(Debug)]
79pub enum PlacementDecision {
80    Immediate(Placement),
81    Queued,
82}
83
84#[derive(Debug)]
85pub struct PlacementEffects {
86    pub decision: PlacementDecision,
87    pub released: Vec<Placement>,
88}
89
90#[derive(Debug, Clone, PartialEq, Eq)]
91pub struct WorkerTopology {
92    pub worker_id: usize,
93    pub scheduler_ids: Vec<usize>,
94}
95
96pub trait PlacementPolicy<Request> {
97    type Metadata;
98    type Observation;
99
100    fn place(
101        &mut self,
102        request: &Request,
103        metadata: Self::Metadata,
104        session_id: Option<String>,
105        now_ms: f64,
106    ) -> Result<PlacementEffects>;
107    fn observe(&mut self, observation: Self::Observation, now_ms: f64) -> Result<Vec<Placement>>;
108    fn cancel_pending(&mut self, request_id: Uuid) -> bool;
109    fn request_terminal(&mut self, request_id: Uuid, now_ms: f64) -> Result<Vec<Placement>>;
110    fn prefill_completed(&mut self, request_id: Uuid, now_ms: f64) -> Result<Vec<Placement>>;
111    fn pending_count(&self) -> usize;
112    fn worker_ready(&mut self, worker: WorkerTopology, now_ms: f64) -> Result<Vec<Placement>>;
113    fn worker_draining(&mut self, worker: WorkerTopology, now_ms: f64) -> Result<Vec<Placement>>;
114    fn worker_removed(&mut self, worker: WorkerTopology, now_ms: f64) -> Result<Vec<Placement>>;
115    fn topology_settled(&mut self, now_ms: f64) -> Result<Vec<Placement>>;
116}
117
118#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
119pub(crate) struct EngineProgress {
120    pub(crate) made_progress: bool,
121    pub(crate) had_raw_observations: bool,
122}
123
124pub trait EngineEventBatch: Default {
125    fn is_empty(&self) -> bool;
126    fn append(&mut self, other: Self);
127}
128
129impl EngineEventBatch for () {
130    #[inline]
131    fn is_empty(&self) -> bool {
132        true
133    }
134
135    #[inline]
136    fn append(&mut self, _other: Self) {}
137}
138
139#[derive(Debug, Default)]
140pub struct NoEngineEvents;
aisimulate_core/replay/core/mod.rs

aisimulate_core/replay/core/
mod.rs
