1use crate::cron_schedule::{
45 apply_one_shot_lead, apply_recurring_jitter, next_fire_after, CronEntry, CronStore,
46};
47use arc_swap::ArcSwap;
48use async_trait::async_trait;
49use chrono::{TimeZone, Timelike, Utc};
50use nexo_config::types::cron_jitter::CronJitterConfig;
51use std::sync::Arc;
52use std::time::Duration;
53use tokio_util::sync::CancellationToken;
54
55pub const DEFAULT_TICK_INTERVAL_SECS: u64 = 5;
59
60#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62pub struct OneShotRetryPolicy {
63 pub max_retries: u32,
66 pub base_backoff_secs: u64,
70 pub max_backoff_secs: u64,
72}
73
74impl OneShotRetryPolicy {
75 pub fn retry_delay_secs(&self, attempt: u32) -> u64 {
76 let base = self.base_backoff_secs.max(1);
77 let pow = attempt.saturating_sub(1).min(31);
78 let factor = 1u64 << pow;
79 base.saturating_mul(factor)
80 .min(self.max_backoff_secs.max(1))
81 }
82}
83
84impl Default for OneShotRetryPolicy {
85 fn default() -> Self {
86 Self {
87 max_retries: 3,
88 base_backoff_secs: 30,
89 max_backoff_secs: 1800,
90 }
91 }
92}
93
94#[async_trait]
97pub trait CronDispatcher: Send + Sync {
98 async fn fire(&self, entry: &CronEntry) -> anyhow::Result<()>;
99}
100
101pub struct LoggingCronDispatcher;
105
106#[async_trait]
107impl CronDispatcher for LoggingCronDispatcher {
108 async fn fire(&self, entry: &CronEntry) -> anyhow::Result<()> {
109 tracing::info!(
110 id = %entry.id,
111 binding_id = %entry.binding_id,
112 cron = %entry.cron_expr,
113 recurring = entry.recurring,
114 channel = ?entry.channel,
115 prompt_chars = entry.prompt.chars().count(),
116 "[cron] fired (logging dispatcher fallback)"
117 );
118 Ok(())
119 }
120}
121
122#[derive(Debug, Clone, PartialEq, Eq)]
126pub enum FireOutcome {
127 Advanced { id: String, new_next_fire_at: i64 },
130 OneShotDeleted { id: String },
132 OneShotRetryScheduled {
134 id: String,
135 retry_at: i64,
136 attempt: u32,
137 max_retries: u32,
138 error: String,
139 },
140 OneShotDroppedAfterRetries {
143 id: String,
144 attempts: u32,
145 max_retries: u32,
146 error: String,
147 },
148 DispatcherFailed { id: String, error: String },
152 NextFireUnknown { id: String, error: String },
156}
157
158pub struct CronRunner {
159 store: Arc<dyn CronStore>,
160 dispatcher: Arc<dyn CronDispatcher>,
161 tick_interval: Duration,
162 one_shot_retry: OneShotRetryPolicy,
163 jitter_cfg: Arc<ArcSwap<CronJitterConfig>>,
168}
169
170impl CronRunner {
171 pub fn new(store: Arc<dyn CronStore>, dispatcher: Arc<dyn CronDispatcher>) -> Self {
172 Self {
173 store,
174 dispatcher,
175 tick_interval: Duration::from_secs(DEFAULT_TICK_INTERVAL_SECS),
176 one_shot_retry: OneShotRetryPolicy::default(),
177 jitter_cfg: Arc::new(ArcSwap::from_pointee(CronJitterConfig::default())),
178 }
179 }
180
181 pub fn with_tick_interval(mut self, interval: Duration) -> Self {
182 self.tick_interval = interval;
183 self
184 }
185
186 pub fn with_one_shot_retry_policy(mut self, policy: OneShotRetryPolicy) -> Self {
187 self.one_shot_retry = policy;
188 self
189 }
190
191 pub fn with_jitter_pct(mut self, pct: u32) -> Self {
196 let cfg = CronJitterConfig::from_legacy_pct(pct);
197 self.jitter_cfg = Arc::new(ArcSwap::from_pointee(cfg));
198 self
199 }
200
201 pub fn with_jitter_cfg(mut self, cfg: Arc<ArcSwap<CronJitterConfig>>) -> Self {
203 self.jitter_cfg = cfg;
204 self
205 }
206
207 pub async fn tick_once(&self, now_unix: i64) -> Vec<FireOutcome> {
210 let cfg = self.jitter_cfg.load_full();
214 if !cfg.enabled {
215 tracing::debug!("[cron] killswitch off — skipping tick");
216 return Vec::new();
217 }
218
219 let due = match self.store.due_at(now_unix).await {
220 Ok(due) => due,
221 Err(e) => {
222 tracing::warn!(error = %e, "[cron] due_at query failed; skipping tick");
223 return Vec::new();
224 }
225 };
226 let mut outcomes = Vec::with_capacity(due.len());
227 for entry in due {
228 let id = entry.id.clone();
229 let dispatch_err = match self.dispatcher.fire(&entry).await {
230 Ok(()) => None,
231 Err(e) => {
232 tracing::warn!(
233 id = %id,
234 binding_id = %entry.binding_id,
235 error = %e,
236 "[cron] dispatcher failed; advancing state anyway to avoid re-fire loop"
237 );
238 Some(e.to_string())
239 }
240 };
241
242 if entry.recurring {
243 match next_fire_after(&entry.cron_expr, now_unix) {
244 Ok(new_next) => {
245 let following =
246 next_fire_after(&entry.cron_expr, new_next).unwrap_or(new_next);
247 let new_next =
248 apply_recurring_jitter(new_next, following, now_unix, &entry.id, &cfg);
249 if let Err(e) = self
250 .store
251 .advance_after_fire(&entry.id, new_next, now_unix)
252 .await
253 {
254 tracing::error!(
255 id = %id,
256 error = %e,
257 "[cron] advance_after_fire failed; entry will likely re-fire next tick"
258 );
259 outcomes.push(FireOutcome::NextFireUnknown {
260 id,
261 error: e.to_string(),
262 });
263 continue;
264 }
265 if let Some(err) = dispatch_err {
266 outcomes.push(FireOutcome::DispatcherFailed { id, error: err });
267 } else {
268 outcomes.push(FireOutcome::Advanced {
269 id,
270 new_next_fire_at: new_next,
271 });
272 }
273 }
274 Err(e) => {
275 tracing::error!(
276 id = %id,
277 error = %e,
278 "[cron] next-fire compute failed; leaving entry as-is for operator"
279 );
280 outcomes.push(FireOutcome::NextFireUnknown {
281 id,
282 error: e.to_string(),
283 });
284 }
285 }
286 } else {
287 if let Some(err) = dispatch_err {
288 let next_attempt = entry.failure_count.saturating_add(1);
289 if entry.failure_count < self.one_shot_retry.max_retries {
290 let delay = self.one_shot_retry.retry_delay_secs(next_attempt);
291 let target = now_unix.saturating_add(delay as i64);
292 let target_minute = Utc
296 .timestamp_opt(target, 0)
297 .single()
298 .map(|dt| dt.minute())
299 .unwrap_or(0);
300 let retry_at =
301 apply_one_shot_lead(target, now_unix, &entry.id, &cfg, target_minute);
302 match self
303 .store
304 .schedule_one_shot_retry(&entry.id, retry_at, now_unix)
305 .await
306 {
307 Ok(attempt) => {
308 tracing::warn!(
309 id = %id,
310 binding_id = %entry.binding_id,
311 attempt = attempt,
312 max_retries = self.one_shot_retry.max_retries,
313 retry_at = retry_at,
314 error = %err,
315 "[cron] one-shot dispatch failed; retry scheduled"
316 );
317 outcomes.push(FireOutcome::OneShotRetryScheduled {
318 id,
319 retry_at,
320 attempt,
321 max_retries: self.one_shot_retry.max_retries,
322 error: err,
323 });
324 }
325 Err(e) => {
326 tracing::error!(
327 id = %id,
328 error = %e,
329 "[cron] one-shot retry schedule failed; entry may re-fire unexpectedly"
330 );
331 outcomes.push(FireOutcome::NextFireUnknown {
332 id,
333 error: e.to_string(),
334 });
335 }
336 }
337 continue;
338 }
339
340 if let Err(e) = self.store.delete(&entry.id).await {
341 tracing::error!(
342 id = %id,
343 error = %e,
344 "[cron] one-shot retry budget exhausted but delete failed"
345 );
346 outcomes.push(FireOutcome::NextFireUnknown {
347 id,
348 error: e.to_string(),
349 });
350 continue;
351 }
352
353 let attempts = entry.failure_count.saturating_add(1);
354 tracing::error!(
355 id = %id,
356 binding_id = %entry.binding_id,
357 attempts = attempts,
358 max_retries = self.one_shot_retry.max_retries,
359 error = %err,
360 "[cron] one-shot dispatch failed; retry budget exhausted, dropping entry"
361 );
362 outcomes.push(FireOutcome::OneShotDroppedAfterRetries {
363 id,
364 attempts,
365 max_retries: self.one_shot_retry.max_retries,
366 error: err,
367 });
368 continue;
369 }
370
371 if let Err(e) = self.store.delete(&entry.id).await {
372 tracing::error!(
373 id = %id,
374 error = %e,
375 "[cron] one-shot delete failed; entry may re-fire next tick"
376 );
377 outcomes.push(FireOutcome::NextFireUnknown {
378 id,
379 error: e.to_string(),
380 });
381 continue;
382 }
383 outcomes.push(FireOutcome::OneShotDeleted { id });
384 }
385 }
386 outcomes
387 }
388
389 pub async fn run(self: Arc<Self>, cancel: CancellationToken) {
392 tracing::info!(
393 tick_interval_secs = self.tick_interval.as_secs(),
394 "[cron] runner started"
395 );
396 loop {
397 tokio::select! {
398 _ = cancel.cancelled() => {
399 tracing::info!("[cron] runner cancelled");
400 break;
401 }
402 _ = tokio::time::sleep(self.tick_interval) => {
403 let now = chrono::Utc::now().timestamp();
404 let _ = self.tick_once(now).await;
405 }
406 }
407 }
408 }
409}
410
411#[cfg(test)]
412mod tests {
413 use super::*;
414 use crate::cron_schedule::{build_new_entry, SqliteCronStore};
415 use std::sync::Mutex;
416
417 #[derive(Default)]
418 struct FakeDispatcher {
419 fires: Mutex<Vec<String>>,
420 force_error: Mutex<Option<String>>,
421 }
422
423 impl FakeDispatcher {
424 fn new() -> Arc<Self> {
425 Arc::new(Self::default())
426 }
427 fn force_err(&self, msg: &str) {
428 *self.force_error.lock().unwrap() = Some(msg.to_string());
429 }
430 fn captured(&self) -> Vec<String> {
431 self.fires.lock().unwrap().clone()
432 }
433 }
434
435 #[async_trait]
436 impl CronDispatcher for FakeDispatcher {
437 async fn fire(&self, entry: &CronEntry) -> anyhow::Result<()> {
438 self.fires.lock().unwrap().push(entry.id.clone());
439 if let Some(msg) = self.force_error.lock().unwrap().clone() {
440 anyhow::bail!(msg);
441 }
442 Ok(())
443 }
444 }
445
446 async fn populated_store(recurring: bool, cron: &str) -> (Arc<dyn CronStore>, String) {
447 let store: Arc<dyn CronStore> = Arc::new(SqliteCronStore::open_memory().await.unwrap());
448 let mut e = build_new_entry(
449 &store,
450 "whatsapp:default",
451 cron,
452 "ping",
453 None,
454 recurring,
455 None,
456 None,
457 None,
458 None,
459 )
460 .await
461 .unwrap();
462 e.next_fire_at = 1_700_000_000;
463 let id = e.id.clone();
464 store.insert(&e).await.unwrap();
465 (store, id)
466 }
467
468 #[tokio::test]
469 async fn tick_advances_recurring_entry() {
470 let (store, id) = populated_store(true, "*/5 * * * *").await;
471 let dispatcher = FakeDispatcher::new();
472 let runner = CronRunner::new(store.clone(), dispatcher.clone());
473 let outcomes = runner.tick_once(1_700_000_500).await;
474 assert_eq!(outcomes.len(), 1);
475 match &outcomes[0] {
476 FireOutcome::Advanced {
477 id: out_id,
478 new_next_fire_at,
479 } => {
480 assert_eq!(out_id, &id);
481 assert!(*new_next_fire_at > 1_700_000_500);
482 }
483 other => panic!("expected Advanced, got {other:?}"),
484 }
485 let updated = store.get(&id).await.unwrap();
487 assert!(updated.next_fire_at > 1_700_000_500);
488 assert_eq!(updated.last_fired_at, Some(1_700_000_500));
489 assert_eq!(dispatcher.captured(), vec![id]);
491 }
492
493 #[tokio::test]
494 async fn tick_deletes_one_shot_after_fire() {
495 let (store, id) = populated_store(false, "*/5 * * * *").await;
496 let dispatcher = FakeDispatcher::new();
497 let runner = CronRunner::new(store.clone(), dispatcher.clone());
498 let outcomes = runner.tick_once(1_700_000_500).await;
499 assert_eq!(outcomes.len(), 1);
500 assert!(
501 matches!(&outcomes[0], FireOutcome::OneShotDeleted { id: out_id } if out_id == &id)
502 );
503 assert!(store.get(&id).await.is_err());
505 assert_eq!(dispatcher.captured().len(), 1);
506 }
507
508 #[tokio::test]
509 async fn tick_skips_paused_entries() {
510 let (store, id) = populated_store(true, "*/5 * * * *").await;
511 store.set_paused(&id, true).await.unwrap();
512 let dispatcher = FakeDispatcher::new();
513 let runner = CronRunner::new(store.clone(), dispatcher.clone());
514 let outcomes = runner.tick_once(1_700_000_500).await;
515 assert!(outcomes.is_empty(), "paused entries must not fire");
516 assert!(dispatcher.captured().is_empty());
517 }
518
519 #[tokio::test]
520 async fn tick_skips_future_entries() {
521 let (store, _id) = populated_store(true, "0 9 * * *").await; let entries = store.list_by_binding("whatsapp:default").await.unwrap();
524 let id = entries[0].id.clone();
525 store
526 .advance_after_fire(&id, 1_700_999_999, 0)
527 .await
528 .unwrap();
529 let dispatcher = FakeDispatcher::new();
530 let runner = CronRunner::new(store.clone(), dispatcher.clone());
531 let outcomes = runner.tick_once(1_700_000_500).await;
532 assert!(outcomes.is_empty(), "future entries must not fire");
533 }
534
535 #[tokio::test]
536 async fn dispatcher_failure_advances_state_anyway() {
537 let (store, id) = populated_store(true, "*/5 * * * *").await;
538 let dispatcher = FakeDispatcher::new();
539 dispatcher.force_err("simulated");
540 let runner = CronRunner::new(store.clone(), dispatcher.clone());
541 let outcomes = runner.tick_once(1_700_000_500).await;
542 assert_eq!(outcomes.len(), 1);
543 assert!(matches!(
544 &outcomes[0],
545 FireOutcome::DispatcherFailed { id: out_id, error } if out_id == &id && error.contains("simulated")
546 ));
547 let updated = store.get(&id).await.unwrap();
550 assert!(updated.next_fire_at > 1_700_000_500);
551 }
552
553 #[tokio::test]
554 async fn dispatcher_failure_on_one_shot_schedules_retry() {
555 let (store, id) = populated_store(false, "*/5 * * * *").await;
556 let dispatcher = FakeDispatcher::new();
557 dispatcher.force_err("boom");
558 let runner = CronRunner::new(store.clone(), dispatcher.clone());
559 let outcomes = runner.tick_once(1_700_000_500).await;
560 assert!(matches!(
561 &outcomes[0],
562 FireOutcome::OneShotRetryScheduled { id: out_id, attempt: 1, .. } if out_id == &id
563 ));
564 let updated = store.get(&id).await.unwrap();
566 assert_eq!(updated.failure_count, 1);
567 assert!(updated.next_fire_at > 1_700_000_500);
568 }
569
570 #[tokio::test]
571 async fn one_shot_drops_after_retry_budget_exhausted() {
572 let (store, id) = populated_store(false, "*/5 * * * *").await;
573 let dispatcher = FakeDispatcher::new();
574 dispatcher.force_err("boom");
575 let runner = CronRunner::new(store.clone(), dispatcher.clone()).with_one_shot_retry_policy(
576 OneShotRetryPolicy {
577 max_retries: 1,
578 base_backoff_secs: 10,
579 max_backoff_secs: 60,
580 },
581 );
582
583 let first = runner.tick_once(1_700_000_500).await;
585 assert!(matches!(
586 &first[0],
587 FireOutcome::OneShotRetryScheduled { id: out_id, attempt: 1, .. } if out_id == &id
588 ));
589 let scheduled = store.get(&id).await.unwrap();
590 assert_eq!(scheduled.failure_count, 1);
591
592 let second = runner.tick_once(scheduled.next_fire_at + 1).await;
594 assert!(matches!(
595 &second[0],
596 FireOutcome::OneShotDroppedAfterRetries {
597 id: out_id,
598 attempts: 2,
599 max_retries: 1,
600 ..
601 } if out_id == &id
602 ));
603 assert!(store.get(&id).await.is_err(), "entry should be deleted");
604 }
605
606 #[tokio::test]
607 async fn many_due_entries_all_fire_in_one_tick() {
608 let store: Arc<dyn CronStore> = Arc::new(SqliteCronStore::open_memory().await.unwrap());
609 for i in 0..5 {
610 let mut e = build_new_entry(
611 &store,
612 "whatsapp:default",
613 "*/5 * * * *",
614 &format!("ping-{i}"),
615 None,
616 true,
617 None,
618 None,
619 None,
620 None,
621 )
622 .await
623 .unwrap();
624 e.next_fire_at = 1_700_000_000;
625 store.insert(&e).await.unwrap();
626 }
627 let dispatcher = FakeDispatcher::new();
628 let runner = CronRunner::new(store.clone(), dispatcher.clone());
629 let outcomes = runner.tick_once(1_700_000_500).await;
630 assert_eq!(outcomes.len(), 5);
631 assert_eq!(dispatcher.captured().len(), 5);
632 let listed = store.list_by_binding("whatsapp:default").await.unwrap();
634 assert!(listed.iter().all(|e| e.next_fire_at > 1_700_000_500));
635 }
636
637 #[tokio::test]
638 async fn run_loop_terminates_on_cancel() {
639 let store: Arc<dyn CronStore> = Arc::new(SqliteCronStore::open_memory().await.unwrap());
640 let dispatcher = FakeDispatcher::new();
641 let runner = Arc::new(
642 CronRunner::new(store, dispatcher.clone())
643 .with_tick_interval(Duration::from_millis(20)),
644 );
645 let cancel = CancellationToken::new();
646 let cancel2 = cancel.clone();
647 let handle = tokio::spawn(async move {
648 runner.run(cancel2).await;
649 });
650 tokio::time::sleep(Duration::from_millis(80)).await;
651 cancel.cancel();
652 tokio::time::timeout(Duration::from_secs(1), handle)
654 .await
655 .expect("runner did not terminate after cancel")
656 .expect("runner task panicked");
657 }
658
659 #[tokio::test]
660 async fn logging_dispatcher_returns_ok() {
661 let entry = CronEntry {
662 id: "x".into(),
663 binding_id: "wp:def".into(),
664 cron_expr: "*/5 * * * *".into(),
665 prompt: "ping".into(),
666 channel: None,
667 model_provider: None,
668 model_name: None,
669 recurring: true,
670 created_at: 0,
671 next_fire_at: 0,
672 last_fired_at: None,
673 failure_count: 0,
674 paused: false,
675 permanent: false,
676 recipient: None,
677 tenant_id: None,
678 };
679 assert!(LoggingCronDispatcher.fire(&entry).await.is_ok());
680 }
681
682 #[tokio::test]
685 async fn killswitch_off_skips_dispatch_and_keeps_entry() {
686 use arc_swap::ArcSwap;
687 use nexo_config::types::cron_jitter::CronJitterConfig;
688
689 let (store, id) = populated_store(true, "*/5 * * * *").await;
690 let dispatcher = FakeDispatcher::new();
691
692 let cfg = Arc::new(ArcSwap::from_pointee(CronJitterConfig {
693 enabled: false,
694 ..CronJitterConfig::default()
695 }));
696 let runner =
697 CronRunner::new(store.clone(), dispatcher.clone()).with_jitter_cfg(cfg.clone());
698
699 let outcomes = runner.tick_once(1_700_000_500).await;
700 assert!(
701 outcomes.is_empty(),
702 "killswitch off must short-circuit tick"
703 );
704 assert!(
705 dispatcher.captured().is_empty(),
706 "dispatcher must not fire while killswitch off"
707 );
708 let stored = store.get(&id).await.unwrap();
710 assert_eq!(stored.last_fired_at, None);
711
712 cfg.store(Arc::new(CronJitterConfig::default()));
714 let outcomes = runner.tick_once(1_700_000_500).await;
715 assert_eq!(outcomes.len(), 1);
716 assert_eq!(dispatcher.captured().len(), 1);
717 }
718}