use std::sync::Arc;
use std::time::Duration;
use chrono::{DateTime, Utc};
use chronon_core::models::{Job, MisfirePolicy, Run, ScheduleKind};
use chronon_core::store::SchedulerStore;
use chronon_core::Result;
use chronon_telemetry::TelemetrySink;
use tokio::sync::Notify;
use tokio::time::sleep;
use crate::cron::CronExpr;
use crate::env::env_flag;
use crate::partition_assigner::PartitionAssigner;
use crate::partitioning::{self, job_execution_pool_id};
async fn count_active_runs(store: &Arc<dyn SchedulerStore>, job_id: &str) -> Result<i32> {
let runs = store.list_runs_for_job(job_id, 10_000).await?;
Ok(runs
.iter()
.filter(|r| r.status.is_active())
.count() as i32)
}
fn cron_next_run_at(job: &Job, after: DateTime<Utc>) -> Option<DateTime<Utc>> {
job.cron_expr
.as_ref()
.and_then(|expr| CronExpr::parse(expr, job.timezone.as_deref()).ok())
.and_then(|c| c.next_after(after))
}
fn should_enqueue_for_misfire(
scheduled_for: DateTime<Utc>,
now: DateTime<Utc>,
policy: &MisfirePolicy,
) -> bool {
if policy.max_misfire_window_secs == 0 {
return true;
}
let lateness_secs = now
.signed_duration_since(scheduled_for)
.num_seconds()
.max(0) as u64;
if lateness_secs == 0 {
return true;
}
lateness_secs <= policy.max_misfire_window_secs && policy.run_immediately
}
async fn skip_due_job(
store: &Arc<dyn SchedulerStore>,
telemetry: &Arc<dyn TelemetrySink>,
job: &Job,
job_id: &str,
now: DateTime<Utc>,
) -> Result<bool> {
if job.schedule_kind == ScheduleKind::RunOnce {
let _ = store.mark_run_once_completed(job_id, now).await;
}
let next_run_at = if job.schedule_kind == ScheduleKind::Cron {
cron_next_run_at(job, now)
} else {
None
};
if let Err(e) = store
.persist_post_tick_job_state(job_id, next_run_at)
.await
{
telemetry.log_event(
"chronon_scheduler_error",
&[("component", "tick_loop"), ("message", &e.to_string())],
);
}
let _ = store.release_job_tick_claim(job_id).await;
Ok(false)
}
async fn enqueue_due_job(
store: &Arc<dyn SchedulerStore>,
telemetry: &Arc<dyn TelemetrySink>,
instance_id: &str,
job_id: &str,
) -> Result<bool> {
let now = Utc::now();
let claim_id = format!("{instance_id}:{}", uuid::Uuid::new_v4());
let lease_ttl = partitioning::job_claim_lease_ttl_secs();
let Ok(true) = store
.claim_job_for_tick(job_id, &claim_id, now, lease_ttl)
.await
else {
return Ok(false);
};
let Some(job) = store.get_job(job_id).await? else {
let _ = store.release_job_tick_claim(job_id).await;
return Ok(false);
};
let active_count = count_active_runs(store, job_id).await.unwrap_or(0);
if active_count >= job.concurrency {
let _ = store.release_job_tick_claim(job_id).await;
return Ok(false);
}
let scheduled_for = job.next_run_at.unwrap_or(now);
if !should_enqueue_for_misfire(scheduled_for, now, &job.misfire_policy()) {
return skip_due_job(store, telemetry, &job, job_id, now).await;
}
if job.schedule_kind == ScheduleKind::RunOnce {
match store
.try_claim_run_once(job_id, instance_id, now, lease_ttl)
.await
{
Ok(true) => {}
Ok(false) => {
let _ = store.release_job_tick_claim(job_id).await;
return Ok(false);
}
Err(e) => {
telemetry.log_event(
"chronon_scheduler_error",
&[("component", "tick_loop"), ("message", &e.to_string())],
);
let _ = store.release_job_tick_claim(job_id).await;
return Ok(false);
}
}
}
let mut run = Run::for_job(&job.job_id, &job.script_name, scheduled_for);
run.actor_json = job.actor_json.clone();
run.params_json = job.params_json.clone();
run.pool_id = Some(job_execution_pool_id(&job));
let run_id = run.run_id.clone();
if store.create_run(&run).await.is_err() {
if job.schedule_kind == ScheduleKind::RunOnce {
let _ = store
.release_run_once_claim(job_id, instance_id, Utc::now())
.await;
}
let _ = store.release_job_tick_claim(job_id).await;
return Ok(false);
}
if job.schedule_kind == ScheduleKind::RunOnce {
let _ = store.mark_run_once_completed(job_id, Utc::now()).await;
}
telemetry.log_event(
"chronon_scheduler_info",
&[
("component", "tick_loop"),
(
"message",
&format!(
"enqueued run run_id={run_id} job_id={} job_name={} script={}",
job.job_id, job.job_name, job.script_name
),
),
],
);
let next_run_at = if job.schedule_kind == ScheduleKind::Cron {
cron_next_run_at(&job, now)
} else {
None
};
if let Err(e) = store
.persist_post_tick_job_state(job_id, next_run_at)
.await
{
telemetry.log_event(
"chronon_scheduler_error",
&[("component", "tick_loop"), ("message", &e.to_string())],
);
}
Ok(true)
}
#[cfg(test)]
mod misfire_tests {
use super::*;
use chronon_core::models::MisfirePolicy;
#[test]
fn window_zero_always_enqueues() {
let now = Utc::now();
let past = now - chrono::Duration::hours(2);
let policy = MisfirePolicy::default();
assert!(should_enqueue_for_misfire(past, now, &policy));
}
#[test]
fn within_window_requires_run_immediately() {
let now = Utc::now();
let past = now - chrono::Duration::seconds(30);
let skip = MisfirePolicy {
run_immediately: false,
max_misfire_window_secs: 60,
};
let fire = MisfirePolicy {
run_immediately: true,
max_misfire_window_secs: 60,
};
assert!(!should_enqueue_for_misfire(past, now, &skip));
assert!(should_enqueue_for_misfire(past, now, &fire));
}
#[test]
fn outside_window_skips() {
let now = Utc::now();
let past = now - chrono::Duration::seconds(120);
let policy = MisfirePolicy {
run_immediately: true,
max_misfire_window_secs: 60,
};
assert!(!should_enqueue_for_misfire(past, now, &policy));
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TickResult {
pub enqueued: usize,
pub due_count: usize,
}
pub async fn run_one_tick(
store: &Arc<dyn SchedulerStore>,
telemetry: &Arc<dyn TelemetrySink>,
instance_id: &str,
assigner: &PartitionAssigner,
draining: &mut bool,
) -> TickResult {
if env_flag("CHRONON_DISABLE_COORDINATOR") || env_flag("CHRONON_INTERACTIVE") {
sleep(Duration::from_secs(30)).await;
return TickResult {
enqueued: 0,
due_count: 0,
};
}
let tick_ms = partitioning::tick_interval_ms_from_env();
let batch_limit = partitioning::tick_batch_limit_from_env();
telemetry.record_counter(
"chronon_scheduler_ticks",
&[("component", "scheduler")],
1,
);
let owned = assigner.owned_partitions().await;
if owned.is_empty() {
sleep(Duration::from_millis(500)).await;
return TickResult {
enqueued: 0,
due_count: 0,
};
}
let now = Utc::now();
let lookahead_ms: i64 = if *draining { 50 } else { 0 };
let due_until = now + chrono::Duration::milliseconds(lookahead_ms);
let ids = match store
.find_due_job_ids_in_partitions(&owned, due_until, batch_limit)
.await
{
Ok(ids) => ids,
Err(e) => {
telemetry.log_event(
"chronon_scheduler_warn",
&[("component", "tick_loop"), ("message", &e.to_string())],
);
sleep(Duration::from_millis(tick_ms)).await;
return TickResult {
enqueued: 0,
due_count: 0,
};
}
};
if ids.is_empty() {
*draining = false;
let wait = match store.min_next_run_at_in_partitions(&owned).await {
Ok(Some(t)) => {
let buf_ms = 20i64;
let ms = t
.signed_duration_since(now)
.num_milliseconds()
.saturating_sub(buf_ms)
.max(0);
Duration::from_millis(ms as u64).min(Duration::from_secs(24 * 3600))
}
_ => Duration::from_millis(tick_ms),
};
sleep(wait).await;
return TickResult {
enqueued: 0,
due_count: 0,
};
}
let min_next = store
.min_next_run_at_in_partitions(&owned)
.await
.ok()
.flatten();
let soon = min_next
.is_some_and(|t| t <= now + chrono::Duration::milliseconds(250));
*draining = u32::try_from(ids.len()).unwrap_or(u32::MAX) >= batch_limit / 2 || soon;
let due_count = ids.len();
let mut enqueued = 0usize;
for job_id in ids {
if enqueue_due_job(store, telemetry, instance_id, &job_id)
.await
.is_ok_and(|v| v)
{
enqueued += 1;
}
}
if *draining {
tokio::task::yield_now().await;
} else {
sleep(Duration::from_millis(tick_ms)).await;
}
TickResult {
enqueued,
due_count,
}
}
pub async fn run_coordinator_tick_loop(
store: Arc<dyn SchedulerStore>,
telemetry: Arc<dyn TelemetrySink>,
instance_id: String,
assigner: Arc<PartitionAssigner>,
shutdown: Arc<Notify>,
) {
let mut draining = false;
loop {
tokio::select! {
() = shutdown.notified() => break,
_ = run_one_tick(&store, &telemetry, &instance_id, &assigner, &mut draining) => {}
}
}
}