Skip to main content

stateknot_runtime/
tenant_scheduler.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Bounded tenant-scoped durable scheduler worker.
5
6use std::{sync::Arc, time::Duration};
7
8use stateknot_core::{AttemptId, BoxFuture, CancellationSignal, RunId, TenantId};
9use stateknot_store_postgres::{
10    PostgresStore, RunnableRunPageCursor, RunnableRunPageSize, StoreError,
11};
12use thiserror::Error;
13
14use crate::{
15    AgentLoopError, AgentLoopResult, DurableAgentLoop, DurableAgentLoopBuildError,
16    DurableGraphDriverOptions, DurableGraphLifecycleOptions, ExecutableGraphRegistry,
17    GraphLifecycleEvidenceProvider,
18};
19
20const MAX_CLAIM_RETRY_DELAY: Duration = Duration::from_secs(1);
21
22/// Bounded scan and claim policy for one tenant scheduler worker.
23#[derive(Clone, Copy, Debug, Eq, PartialEq)]
24pub struct DurableTenantSchedulerOptions {
25    page_size: RunnableRunPageSize,
26    maximum_pages_per_tick: u8,
27    maximum_claim_attempts: u8,
28    claim_retry_initial_delay: Duration,
29}
30
31impl DurableTenantSchedulerOptions {
32    /// Absolute number of stable-snapshot pages one tick may inspect.
33    pub const HARD_MAXIMUM_PAGES_PER_TICK: u8 = 64;
34    /// Absolute number of identical lease-claim attempts.
35    pub const HARD_MAXIMUM_CLAIM_ATTEMPTS: u8 = 10;
36
37    /// Constructs an explicit resource and retry policy.
38    ///
39    /// # Errors
40    ///
41    /// Rejects zero/excessive page or claim counts, and a zero or
42    /// greater-than-one-second initial retry delay.
43    pub fn new(
44        page_size: RunnableRunPageSize,
45        maximum_pages_per_tick: u8,
46        maximum_claim_attempts: u8,
47        claim_retry_initial_delay: Duration,
48    ) -> Result<Self, DurableTenantSchedulerOptionsError> {
49        if maximum_pages_per_tick == 0 || maximum_pages_per_tick > Self::HARD_MAXIMUM_PAGES_PER_TICK
50        {
51            return Err(DurableTenantSchedulerOptionsError::InvalidPageCount);
52        }
53        if maximum_claim_attempts == 0 || maximum_claim_attempts > Self::HARD_MAXIMUM_CLAIM_ATTEMPTS
54        {
55            return Err(DurableTenantSchedulerOptionsError::InvalidClaimAttempts);
56        }
57        if claim_retry_initial_delay.is_zero() || claim_retry_initial_delay > MAX_CLAIM_RETRY_DELAY
58        {
59            return Err(DurableTenantSchedulerOptionsError::InvalidClaimRetryDelay);
60        }
61        Ok(Self {
62            page_size,
63            maximum_pages_per_tick,
64            maximum_claim_attempts,
65            claim_retry_initial_delay,
66        })
67    }
68
69    /// Returns the bounded decoded candidate page size.
70    #[must_use]
71    pub const fn page_size(self) -> RunnableRunPageSize {
72        self.page_size
73    }
74
75    /// Returns the maximum fixed-cutoff pages inspected per tick.
76    #[must_use]
77    pub const fn maximum_pages_per_tick(self) -> u8 {
78        self.maximum_pages_per_tick
79    }
80
81    /// Returns the maximum identical attempts for one claim.
82    #[must_use]
83    pub const fn maximum_claim_attempts(self) -> u8 {
84        self.maximum_claim_attempts
85    }
86
87    /// Returns the first exponential claim retry delay.
88    #[must_use]
89    pub const fn claim_retry_initial_delay(self) -> Duration {
90        self.claim_retry_initial_delay
91    }
92}
93
94impl Default for DurableTenantSchedulerOptions {
95    fn default() -> Self {
96        Self {
97            page_size: RunnableRunPageSize::new(RunnableRunPageSize::MAX)
98                .expect("provider maximum is a valid page size"),
99            maximum_pages_per_tick: 4,
100            maximum_claim_attempts: 3,
101            claim_retry_initial_delay: Duration::from_millis(25),
102        }
103    }
104}
105
106/// Invalid tenant scheduler policy.
107#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
108#[non_exhaustive]
109pub enum DurableTenantSchedulerOptionsError {
110    /// The page-chain bound was zero or above the hard ceiling.
111    #[error("tenant scheduler page count is invalid")]
112    InvalidPageCount,
113    /// Claim attempts were zero or above the hard ceiling.
114    #[error("tenant scheduler claim attempt count is invalid")]
115    InvalidClaimAttempts,
116    /// Initial claim backoff was zero or above one second.
117    #[error("tenant scheduler claim retry delay is invalid")]
118    InvalidClaimRetryDelay,
119}
120
121/// Bounded observations made while selecting one run.
122#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
123pub struct TenantSchedulerReport {
124    pages_scanned: u8,
125    candidates_scanned: u16,
126    contention_skips: u16,
127    claim_retries: u16,
128}
129
130impl TenantSchedulerReport {
131    /// Returns stable-snapshot pages loaded in this tick.
132    #[must_use]
133    pub const fn pages_scanned(self) -> u8 {
134        self.pages_scanned
135    }
136
137    /// Returns candidates inspected in durable queue order.
138    #[must_use]
139    pub const fn candidates_scanned(self) -> u16 {
140        self.candidates_scanned
141    }
142
143    /// Returns candidates that changed or were owned before claim.
144    #[must_use]
145    pub const fn contention_skips(self) -> u16 {
146        self.contention_skips
147    }
148
149    /// Returns transient claim retries using the same attempt identity.
150    #[must_use]
151    pub const fn claim_retries(self) -> u16 {
152        self.claim_retries
153    }
154}
155
156/// Closed result of one tenant scheduler tick.
157#[derive(Debug)]
158#[non_exhaustive]
159#[allow(clippy::large_enum_variant)]
160pub enum TenantSchedulerOutcome {
161    /// One claimed run reached its next agent-loop boundary.
162    Executed {
163        /// Exact selected run.
164        run_id: RunId,
165        /// Complete bounded loop result.
166        result: AgentLoopResult,
167    },
168    /// One claimed run failed safely; the loop attempted lease cleanup.
169    ExecutionFailed {
170        /// Exact selected run.
171        run_id: RunId,
172        /// Payload-redacted run-local failure.
173        error: AgentLoopError,
174    },
175    /// No claimable run existed in the complete stable snapshot.
176    Idle,
177    /// The configured scan ceiling stopped before the snapshot ended.
178    ScanLimitReached,
179    /// Cooperative shutdown won before a claim.
180    Cancelled,
181}
182
183/// Scheduler outcome plus bounded selection counters.
184#[derive(Debug)]
185pub struct TenantSchedulerTick {
186    outcome: TenantSchedulerOutcome,
187    report: TenantSchedulerReport,
188}
189
190impl TenantSchedulerTick {
191    const fn new(outcome: TenantSchedulerOutcome, report: TenantSchedulerReport) -> Self {
192        Self { outcome, report }
193    }
194
195    /// Returns the tick's closed scheduling result.
196    #[must_use]
197    pub const fn outcome(&self) -> &TenantSchedulerOutcome {
198        &self.outcome
199    }
200
201    /// Returns bounded scan and contention counters.
202    #[must_use]
203    pub const fn report(&self) -> TenantSchedulerReport {
204        self.report
205    }
206
207    /// Consumes the tick into its outcome and report.
208    #[must_use]
209    pub fn into_parts(self) -> (TenantSchedulerOutcome, TenantSchedulerReport) {
210        (self.outcome, self.report)
211    }
212}
213
214/// One tenant-isolated durable scheduler worker.
215///
216/// A tick always scans the provider's `(available_at, run_id)` order under one
217/// fixed database cutoff and claims at most one run. Deployments obtain bounded
218/// concurrency by running a configured number of workers; database fencing
219/// resolves races. Cross-tenant selection intentionally belongs to a separate
220/// fairness layer so storage credentials and queue scans never cross tenant
221/// scope implicitly.
222#[derive(Clone)]
223pub struct DurableTenantScheduler {
224    store: PostgresStore,
225    agent_loop: DurableAgentLoop,
226    options: DurableTenantSchedulerOptions,
227}
228
229impl DurableTenantScheduler {
230    /// Returns shared node-future completion accounting for this scheduler and clones.
231    pub fn execution_activity(&self) -> crate::GraphExecutionActivity {
232        self.agent_loop.execution_activity()
233    }
234
235    /// Builds a tenant worker and its internally consistent durable loop.
236    ///
237    /// # Errors
238    ///
239    /// Returns an exact driver/lifecycle startup binding failure.
240    #[allow(clippy::too_many_arguments)]
241    pub fn new(
242        store: PostgresStore,
243        registry: ExecutableGraphRegistry,
244        evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
245        driver_options: DurableGraphDriverOptions,
246        lifecycle_options: DurableGraphLifecycleOptions,
247        options: DurableTenantSchedulerOptions,
248    ) -> Result<Self, DurableTenantSchedulerBuildError> {
249        let agent_loop = DurableAgentLoop::new(
250            store.clone(),
251            registry,
252            evidence,
253            driver_options,
254            lifecycle_options,
255        )?;
256        Ok(Self {
257            store,
258            agent_loop,
259            options,
260        })
261    }
262
263    /// Returns the immutable tenant scan and claim policy.
264    #[must_use]
265    pub const fn options(&self) -> DurableTenantSchedulerOptions {
266        self.options
267    }
268
269    /// Selects, claims, and executes at most one run for the supplied tenant.
270    ///
271    /// A stable attempt ID is allocated once per candidate and retained across
272    /// transient database retries, preserving lost-acknowledgement convergence.
273    /// Lease contention and candidates that became unavailable are normal scan
274    /// skips, not scheduler failures.
275    pub fn tick(
276        &self,
277        tenant_id: TenantId,
278        shutdown: CancellationSignal,
279    ) -> BoxFuture<'_, Result<TenantSchedulerTick, TenantSchedulerError>> {
280        Box::pin(self.tick_inner(tenant_id, shutdown))
281    }
282
283    async fn tick_inner(
284        &self,
285        tenant_id: TenantId,
286        shutdown: CancellationSignal,
287    ) -> Result<TenantSchedulerTick, TenantSchedulerError> {
288        let mut cursor: Option<RunnableRunPageCursor> = None;
289        let mut report = TenantSchedulerReport::default();
290        loop {
291            if shutdown.is_cancelled() {
292                return Ok(TenantSchedulerTick::new(
293                    TenantSchedulerOutcome::Cancelled,
294                    report,
295                ));
296            }
297            let page = self
298                .store
299                .load_runnable_run_page(&tenant_id, cursor.as_ref(), self.options.page_size)
300                .await?;
301            report.pages_scanned = report.pages_scanned.saturating_add(1);
302            for candidate in page.records() {
303                if shutdown.is_cancelled() {
304                    return Ok(TenantSchedulerTick::new(
305                        TenantSchedulerOutcome::Cancelled,
306                        report,
307                    ));
308                }
309                report.candidates_scanned = report.candidates_scanned.saturating_add(1);
310                let run_id = candidate.run().lifecycle().provenance().run_id();
311                let attempt_id = AttemptId::generate();
312                let Some(lease) = self
313                    .claim_with_retry(&tenant_id, run_id, attempt_id, &mut report)
314                    .await?
315                else {
316                    continue;
317                };
318                let outcome = match self
319                    .agent_loop
320                    .run(lease.fence().clone(), shutdown.clone())
321                    .await
322                {
323                    Ok(result) => TenantSchedulerOutcome::Executed { run_id, result },
324                    Err(error) => TenantSchedulerOutcome::ExecutionFailed { run_id, error },
325                };
326                return Ok(TenantSchedulerTick::new(outcome, report));
327            }
328
329            if !page.has_more() {
330                return Ok(TenantSchedulerTick::new(
331                    TenantSchedulerOutcome::Idle,
332                    report,
333                ));
334            }
335            if report.pages_scanned == self.options.maximum_pages_per_tick {
336                return Ok(TenantSchedulerTick::new(
337                    TenantSchedulerOutcome::ScanLimitReached,
338                    report,
339                ));
340            }
341            cursor = page.next_cursor();
342            if cursor.is_none() {
343                return Err(TenantSchedulerError::RuntimeInvariant);
344            }
345        }
346    }
347
348    async fn claim_with_retry(
349        &self,
350        tenant_id: &TenantId,
351        run_id: RunId,
352        attempt_id: AttemptId,
353        report: &mut TenantSchedulerReport,
354    ) -> Result<Option<stateknot_core::RunLease>, TenantSchedulerError> {
355        let mut attempt = 1_u8;
356        loop {
357            match self.store.claim_lease(tenant_id, run_id, attempt_id).await {
358                Ok(outcome) => return Ok(Some(outcome.lease().clone())),
359                Err(error) if claim_contention(&error) => {
360                    report.contention_skips = report.contention_skips.saturating_add(1);
361                    return Ok(None);
362                }
363                Err(error)
364                    if attempt < self.options.maximum_claim_attempts && error.is_retryable() =>
365                {
366                    report.claim_retries = report.claim_retries.saturating_add(1);
367                    self.claim_backoff(attempt).await;
368                    attempt = attempt.saturating_add(1);
369                }
370                Err(error) => return Err(error.into()),
371            }
372        }
373    }
374
375    async fn claim_backoff(&self, attempt: u8) {
376        let multiplier = 1_u32
377            .checked_shl(u32::from(attempt.saturating_sub(1)))
378            .unwrap_or(u32::MAX);
379        let delay = self
380            .options
381            .claim_retry_initial_delay
382            .checked_mul(multiplier)
383            .unwrap_or(MAX_CLAIM_RETRY_DELAY)
384            .min(MAX_CLAIM_RETRY_DELAY);
385        tokio::time::sleep(delay).await;
386    }
387}
388
389fn claim_contention(error: &StoreError) -> bool {
390    matches!(
391        error,
392        StoreError::LeaseHeld
393            | StoreError::RunNotRunnable
394            | StoreError::RunNotYetAvailable
395            | StoreError::RunQuarantined
396            | StoreError::RunNotFound
397    )
398}
399
400/// Startup failure while building a tenant scheduler worker.
401#[derive(Clone, Debug, Eq, Error, PartialEq)]
402#[non_exhaustive]
403pub enum DurableTenantSchedulerBuildError {
404    /// The internally consistent durable agent loop could not be built.
405    #[error(transparent)]
406    AgentLoop(#[from] DurableAgentLoopBuildError),
407}
408
409/// Payload-redacted scheduler infrastructure failure.
410#[derive(Debug, Error)]
411#[non_exhaustive]
412pub enum TenantSchedulerError {
413    /// `PostgreSQL` rejected or could not complete a scan or claim.
414    #[error(transparent)]
415    Store {
416        /// Exact payload-redacted provider failure.
417        source: Box<StoreError>,
418    },
419    /// A provider page claimed continuation without a usable cursor.
420    #[error("tenant scheduler stable page chain is internally inconsistent")]
421    RuntimeInvariant,
422}
423
424impl From<StoreError> for TenantSchedulerError {
425    fn from(source: StoreError) -> Self {
426        Self::Store {
427            source: Box::new(source),
428        }
429    }
430}
431
432#[cfg(test)]
433mod tests {
434    use super::*;
435
436    #[test]
437    fn options_enforce_bounded_scan_and_stable_claim_retry_limits() {
438        let page = RunnableRunPageSize::new(8).unwrap();
439        assert_eq!(
440            DurableTenantSchedulerOptions::new(page, 0, 1, Duration::from_millis(1)),
441            Err(DurableTenantSchedulerOptionsError::InvalidPageCount)
442        );
443        assert_eq!(
444            DurableTenantSchedulerOptions::new(page, 1, 0, Duration::from_millis(1)),
445            Err(DurableTenantSchedulerOptionsError::InvalidClaimAttempts)
446        );
447        assert_eq!(
448            DurableTenantSchedulerOptions::new(page, 1, 1, Duration::ZERO),
449            Err(DurableTenantSchedulerOptionsError::InvalidClaimRetryDelay)
450        );
451        let options =
452            DurableTenantSchedulerOptions::new(page, 3, 4, Duration::from_millis(50)).unwrap();
453        assert_eq!(options.page_size().get(), 8);
454        assert_eq!(options.maximum_pages_per_tick(), 3);
455        assert_eq!(options.maximum_claim_attempts(), 4);
456    }
457}