1use std::{sync::Arc, time::Duration};
7
8use stateknot_core::{AttemptId, BoxFuture, CancellationSignal, RunId, TenantId};
9use stateknot_store_postgres::{
10 PostgresStore, RunnableRunPageCursor, RunnableRunPageSize, StoreError,
11};
12use thiserror::Error;
13
14use crate::{
15 AgentLoopError, AgentLoopResult, DurableAgentLoop, DurableAgentLoopBuildError,
16 DurableGraphDriverOptions, DurableGraphLifecycleOptions, ExecutableGraphRegistry,
17 GraphLifecycleEvidenceProvider,
18};
19
20const MAX_CLAIM_RETRY_DELAY: Duration = Duration::from_secs(1);
21
22#[derive(Clone, Copy, Debug, Eq, PartialEq)]
24pub struct DurableTenantSchedulerOptions {
25 page_size: RunnableRunPageSize,
26 maximum_pages_per_tick: u8,
27 maximum_claim_attempts: u8,
28 claim_retry_initial_delay: Duration,
29}
30
31impl DurableTenantSchedulerOptions {
32 pub const HARD_MAXIMUM_PAGES_PER_TICK: u8 = 64;
34 pub const HARD_MAXIMUM_CLAIM_ATTEMPTS: u8 = 10;
36
37 pub fn new(
44 page_size: RunnableRunPageSize,
45 maximum_pages_per_tick: u8,
46 maximum_claim_attempts: u8,
47 claim_retry_initial_delay: Duration,
48 ) -> Result<Self, DurableTenantSchedulerOptionsError> {
49 if maximum_pages_per_tick == 0 || maximum_pages_per_tick > Self::HARD_MAXIMUM_PAGES_PER_TICK
50 {
51 return Err(DurableTenantSchedulerOptionsError::InvalidPageCount);
52 }
53 if maximum_claim_attempts == 0 || maximum_claim_attempts > Self::HARD_MAXIMUM_CLAIM_ATTEMPTS
54 {
55 return Err(DurableTenantSchedulerOptionsError::InvalidClaimAttempts);
56 }
57 if claim_retry_initial_delay.is_zero() || claim_retry_initial_delay > MAX_CLAIM_RETRY_DELAY
58 {
59 return Err(DurableTenantSchedulerOptionsError::InvalidClaimRetryDelay);
60 }
61 Ok(Self {
62 page_size,
63 maximum_pages_per_tick,
64 maximum_claim_attempts,
65 claim_retry_initial_delay,
66 })
67 }
68
69 #[must_use]
71 pub const fn page_size(self) -> RunnableRunPageSize {
72 self.page_size
73 }
74
75 #[must_use]
77 pub const fn maximum_pages_per_tick(self) -> u8 {
78 self.maximum_pages_per_tick
79 }
80
81 #[must_use]
83 pub const fn maximum_claim_attempts(self) -> u8 {
84 self.maximum_claim_attempts
85 }
86
87 #[must_use]
89 pub const fn claim_retry_initial_delay(self) -> Duration {
90 self.claim_retry_initial_delay
91 }
92}
93
94impl Default for DurableTenantSchedulerOptions {
95 fn default() -> Self {
96 Self {
97 page_size: RunnableRunPageSize::new(RunnableRunPageSize::MAX)
98 .expect("provider maximum is a valid page size"),
99 maximum_pages_per_tick: 4,
100 maximum_claim_attempts: 3,
101 claim_retry_initial_delay: Duration::from_millis(25),
102 }
103 }
104}
105
106#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
108#[non_exhaustive]
109pub enum DurableTenantSchedulerOptionsError {
110 #[error("tenant scheduler page count is invalid")]
112 InvalidPageCount,
113 #[error("tenant scheduler claim attempt count is invalid")]
115 InvalidClaimAttempts,
116 #[error("tenant scheduler claim retry delay is invalid")]
118 InvalidClaimRetryDelay,
119}
120
121#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
123pub struct TenantSchedulerReport {
124 pages_scanned: u8,
125 candidates_scanned: u16,
126 contention_skips: u16,
127 claim_retries: u16,
128}
129
130impl TenantSchedulerReport {
131 #[must_use]
133 pub const fn pages_scanned(self) -> u8 {
134 self.pages_scanned
135 }
136
137 #[must_use]
139 pub const fn candidates_scanned(self) -> u16 {
140 self.candidates_scanned
141 }
142
143 #[must_use]
145 pub const fn contention_skips(self) -> u16 {
146 self.contention_skips
147 }
148
149 #[must_use]
151 pub const fn claim_retries(self) -> u16 {
152 self.claim_retries
153 }
154}
155
156#[derive(Debug)]
158#[non_exhaustive]
159#[allow(clippy::large_enum_variant)]
160pub enum TenantSchedulerOutcome {
161 Executed {
163 run_id: RunId,
165 result: AgentLoopResult,
167 },
168 ExecutionFailed {
170 run_id: RunId,
172 error: AgentLoopError,
174 },
175 Idle,
177 ScanLimitReached,
179 Cancelled,
181}
182
183#[derive(Debug)]
185pub struct TenantSchedulerTick {
186 outcome: TenantSchedulerOutcome,
187 report: TenantSchedulerReport,
188}
189
190impl TenantSchedulerTick {
191 const fn new(outcome: TenantSchedulerOutcome, report: TenantSchedulerReport) -> Self {
192 Self { outcome, report }
193 }
194
195 #[must_use]
197 pub const fn outcome(&self) -> &TenantSchedulerOutcome {
198 &self.outcome
199 }
200
201 #[must_use]
203 pub const fn report(&self) -> TenantSchedulerReport {
204 self.report
205 }
206
207 #[must_use]
209 pub fn into_parts(self) -> (TenantSchedulerOutcome, TenantSchedulerReport) {
210 (self.outcome, self.report)
211 }
212}
213
214#[derive(Clone)]
223pub struct DurableTenantScheduler {
224 store: PostgresStore,
225 agent_loop: DurableAgentLoop,
226 options: DurableTenantSchedulerOptions,
227}
228
229impl DurableTenantScheduler {
230 pub fn execution_activity(&self) -> crate::GraphExecutionActivity {
232 self.agent_loop.execution_activity()
233 }
234
235 #[allow(clippy::too_many_arguments)]
241 pub fn new(
242 store: PostgresStore,
243 registry: ExecutableGraphRegistry,
244 evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
245 driver_options: DurableGraphDriverOptions,
246 lifecycle_options: DurableGraphLifecycleOptions,
247 options: DurableTenantSchedulerOptions,
248 ) -> Result<Self, DurableTenantSchedulerBuildError> {
249 let agent_loop = DurableAgentLoop::new(
250 store.clone(),
251 registry,
252 evidence,
253 driver_options,
254 lifecycle_options,
255 )?;
256 Ok(Self {
257 store,
258 agent_loop,
259 options,
260 })
261 }
262
263 #[must_use]
265 pub const fn options(&self) -> DurableTenantSchedulerOptions {
266 self.options
267 }
268
269 pub fn tick(
276 &self,
277 tenant_id: TenantId,
278 shutdown: CancellationSignal,
279 ) -> BoxFuture<'_, Result<TenantSchedulerTick, TenantSchedulerError>> {
280 Box::pin(self.tick_inner(tenant_id, shutdown))
281 }
282
283 async fn tick_inner(
284 &self,
285 tenant_id: TenantId,
286 shutdown: CancellationSignal,
287 ) -> Result<TenantSchedulerTick, TenantSchedulerError> {
288 let mut cursor: Option<RunnableRunPageCursor> = None;
289 let mut report = TenantSchedulerReport::default();
290 loop {
291 if shutdown.is_cancelled() {
292 return Ok(TenantSchedulerTick::new(
293 TenantSchedulerOutcome::Cancelled,
294 report,
295 ));
296 }
297 let page = self
298 .store
299 .load_runnable_run_page(&tenant_id, cursor.as_ref(), self.options.page_size)
300 .await?;
301 report.pages_scanned = report.pages_scanned.saturating_add(1);
302 for candidate in page.records() {
303 if shutdown.is_cancelled() {
304 return Ok(TenantSchedulerTick::new(
305 TenantSchedulerOutcome::Cancelled,
306 report,
307 ));
308 }
309 report.candidates_scanned = report.candidates_scanned.saturating_add(1);
310 let run_id = candidate.run().lifecycle().provenance().run_id();
311 let attempt_id = AttemptId::generate();
312 let Some(lease) = self
313 .claim_with_retry(&tenant_id, run_id, attempt_id, &mut report)
314 .await?
315 else {
316 continue;
317 };
318 let outcome = match self
319 .agent_loop
320 .run(lease.fence().clone(), shutdown.clone())
321 .await
322 {
323 Ok(result) => TenantSchedulerOutcome::Executed { run_id, result },
324 Err(error) => TenantSchedulerOutcome::ExecutionFailed { run_id, error },
325 };
326 return Ok(TenantSchedulerTick::new(outcome, report));
327 }
328
329 if !page.has_more() {
330 return Ok(TenantSchedulerTick::new(
331 TenantSchedulerOutcome::Idle,
332 report,
333 ));
334 }
335 if report.pages_scanned == self.options.maximum_pages_per_tick {
336 return Ok(TenantSchedulerTick::new(
337 TenantSchedulerOutcome::ScanLimitReached,
338 report,
339 ));
340 }
341 cursor = page.next_cursor();
342 if cursor.is_none() {
343 return Err(TenantSchedulerError::RuntimeInvariant);
344 }
345 }
346 }
347
348 async fn claim_with_retry(
349 &self,
350 tenant_id: &TenantId,
351 run_id: RunId,
352 attempt_id: AttemptId,
353 report: &mut TenantSchedulerReport,
354 ) -> Result<Option<stateknot_core::RunLease>, TenantSchedulerError> {
355 let mut attempt = 1_u8;
356 loop {
357 match self.store.claim_lease(tenant_id, run_id, attempt_id).await {
358 Ok(outcome) => return Ok(Some(outcome.lease().clone())),
359 Err(error) if claim_contention(&error) => {
360 report.contention_skips = report.contention_skips.saturating_add(1);
361 return Ok(None);
362 }
363 Err(error)
364 if attempt < self.options.maximum_claim_attempts && error.is_retryable() =>
365 {
366 report.claim_retries = report.claim_retries.saturating_add(1);
367 self.claim_backoff(attempt).await;
368 attempt = attempt.saturating_add(1);
369 }
370 Err(error) => return Err(error.into()),
371 }
372 }
373 }
374
375 async fn claim_backoff(&self, attempt: u8) {
376 let multiplier = 1_u32
377 .checked_shl(u32::from(attempt.saturating_sub(1)))
378 .unwrap_or(u32::MAX);
379 let delay = self
380 .options
381 .claim_retry_initial_delay
382 .checked_mul(multiplier)
383 .unwrap_or(MAX_CLAIM_RETRY_DELAY)
384 .min(MAX_CLAIM_RETRY_DELAY);
385 tokio::time::sleep(delay).await;
386 }
387}
388
389fn claim_contention(error: &StoreError) -> bool {
390 matches!(
391 error,
392 StoreError::LeaseHeld
393 | StoreError::RunNotRunnable
394 | StoreError::RunNotYetAvailable
395 | StoreError::RunQuarantined
396 | StoreError::RunNotFound
397 )
398}
399
400#[derive(Clone, Debug, Eq, Error, PartialEq)]
402#[non_exhaustive]
403pub enum DurableTenantSchedulerBuildError {
404 #[error(transparent)]
406 AgentLoop(#[from] DurableAgentLoopBuildError),
407}
408
409#[derive(Debug, Error)]
411#[non_exhaustive]
412pub enum TenantSchedulerError {
413 #[error(transparent)]
415 Store {
416 source: Box<StoreError>,
418 },
419 #[error("tenant scheduler stable page chain is internally inconsistent")]
421 RuntimeInvariant,
422}
423
424impl From<StoreError> for TenantSchedulerError {
425 fn from(source: StoreError) -> Self {
426 Self::Store {
427 source: Box::new(source),
428 }
429 }
430}
431
432#[cfg(test)]
433mod tests {
434 use super::*;
435
436 #[test]
437 fn options_enforce_bounded_scan_and_stable_claim_retry_limits() {
438 let page = RunnableRunPageSize::new(8).unwrap();
439 assert_eq!(
440 DurableTenantSchedulerOptions::new(page, 0, 1, Duration::from_millis(1)),
441 Err(DurableTenantSchedulerOptionsError::InvalidPageCount)
442 );
443 assert_eq!(
444 DurableTenantSchedulerOptions::new(page, 1, 0, Duration::from_millis(1)),
445 Err(DurableTenantSchedulerOptionsError::InvalidClaimAttempts)
446 );
447 assert_eq!(
448 DurableTenantSchedulerOptions::new(page, 1, 1, Duration::ZERO),
449 Err(DurableTenantSchedulerOptionsError::InvalidClaimRetryDelay)
450 );
451 let options =
452 DurableTenantSchedulerOptions::new(page, 3, 4, Duration::from_millis(50)).unwrap();
453 assert_eq!(options.page_size().get(), 8);
454 assert_eq!(options.maximum_pages_per_tick(), 3);
455 assert_eq!(options.maximum_claim_attempts(), 4);
456 }
457}