mod key;
mod leader;
pub(crate) use key::{CoalesceGuard, CoalesceResult};
pub use key::{CoalesceKey, RequestCoalescer};
use leader::{recv_leader_signal, LeaderSignal};
use crate::domain::models::{Task, TaskStatus};
use crate::domain::repositories::scrape_result_repository::ScrapeResultRepository;
use crate::domain::repositories::task_repository::TaskRepository;
use crate::utils::coalesce::CoalesceSignal;
use crate::utils::url_redaction::redact_url_for_log;
use anyhow::Result;
use chrono::Utc;
use log::{error, info, warn};
use serde_json::json;
use std::sync::Arc;
#[derive(Clone)]
pub struct CoalesceCoordinator {
repository: Arc<dyn TaskRepository>,
result_repository: Arc<dyn ScrapeResultRepository>,
request_coalescer: Arc<RequestCoalescer>,
}
impl CoalesceCoordinator {
#[must_use]
pub fn new(
repository: Arc<dyn TaskRepository>,
result_repository: Arc<dyn ScrapeResultRepository>,
request_coalescer: Arc<RequestCoalescer>,
) -> Self {
Self {
repository,
result_repository,
request_coalescer,
}
}
pub(crate) async fn try_coalesce(
&self,
key: &CoalesceKey,
task: &Task,
worker_id: crate::domain::id::Id,
) -> Result<Option<CoalesceGuard>> {
if key.team_id() != task.team_id {
debug_assert_eq!(
key.team_id(),
task.team_id,
"coalesce key tenant must match task tenant"
);
error!(
"coalesce key tenant mismatch: key team {} vs task team {} (task {})",
key.team_id(),
task.team_id,
task.id
);
anyhow::bail!(
"coalesce key tenant mismatch: key team {} != task team {}",
key.team_id(),
task.team_id
);
}
match self.request_coalescer.try_start(key, task.id) {
CoalesceResult::Proceed(g) => Ok(Some(g)),
CoalesceResult::Wait(rx, leader_task_id) => {
self.wait_for_leader(key, task, worker_id, rx, leader_task_id)
.await
}
}
}
async fn wait_for_leader(
&self,
key: &CoalesceKey,
task: &Task,
worker_id: crate::domain::id::Id,
mut rx: tokio::sync::broadcast::Receiver<CoalesceSignal>,
leader_task_id: crate::domain::id::Id,
) -> Result<Option<CoalesceGuard>> {
info!(
"URL {} (team {}) already in-flight (leader task {}), task {} waiting for coalesce",
redact_url_for_log(key.url()),
key.team_id(),
leader_task_id,
task.id
);
const COALESCE_RECV_TIMEOUT_SECS: u64 = 45;
let signal = match recv_leader_signal(&mut rx, COALESCE_RECV_TIMEOUT_SECS).await {
LeaderSignal::Signal(sig) => sig,
LeaderSignal::TimedOut => {
error!(
"Coalesce wait for task {} timed out after {}s (leader task {}); \
rescheduling self instead of failing leader",
task.id, COALESCE_RECV_TIMEOUT_SECS, leader_task_id
);
return self
.reschedule_or_fail(task, worker_id, "coalesce wait timeout")
.await;
}
LeaderSignal::Closed => {
warn!(
"Coalesce channel closed for task {} (leader task {} likely panicked); \
rescheduling self",
task.id, leader_task_id
);
return self
.reschedule_or_fail(task, worker_id, "coalesce channel closed")
.await;
}
LeaderSignal::Lagged(n) => {
warn!(
"Coalesce receiver for task {} lagged by {} messages; rescheduling self",
task.id, n
);
return self
.reschedule_or_fail(task, worker_id, "coalesce receiver lagged")
.await;
}
};
match signal {
CoalesceSignal::Purged => {
warn!(
"Coalesce entry for task {} purged as stale (leader task {}); \
rescheduling self without result lookup",
task.id, leader_task_id
);
self.reschedule_or_fail(task, worker_id, "coalesce entry purged")
.await
}
CoalesceSignal::Completed { success: false } => {
warn!(
"Coalesce leader task {} reported failure for waiter task {}; \
rescheduling self without result lookup",
leader_task_id, task.id
);
self.reschedule_or_fail(task, worker_id, "leader reported failure")
.await
}
CoalesceSignal::Completed { success: true } => {
if self
.adopt_leader_result(task, leader_task_id, worker_id)
.await?
{
Ok(None)
} else {
self.reschedule_or_fail(task, worker_id, "leader result not yet available")
.await
}
}
}
}
async fn reschedule_or_fail(
&self,
task: &Task,
_worker_id: crate::domain::id::Id,
reason: &str,
) -> Result<Option<CoalesceGuard>> {
const MAX_COALESCE_RESCHEDULE: u32 = 3;
let reschedule_count = task
.payload
.get("coalesce_reschedule_count")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32;
if reschedule_count >= MAX_COALESCE_RESCHEDULE {
error!(
"Coalesced task {} exceeded max reschedule count ({}) after [{}], marking failed",
task.id, MAX_COALESCE_RESCHEDULE, reason
);
self.repository
.mark_failed_with_reason(
task,
&format!("Coalesce reschedule limit exceeded after [{reason}]"),
)
.await?;
return Ok(None);
}
warn!(
"Coalesced task {} rescheduling after [{}] (attempt {}/{})",
task.id,
reason,
reschedule_count + 1,
MAX_COALESCE_RESCHEDULE
);
let mut updated = task.clone();
updated.scheduled_at = Some(Utc::now() + chrono::Duration::seconds(5));
updated.status = TaskStatus::Queued;
if !updated.payload.is_object() {
updated.payload = json!({});
}
if let Some(obj) = updated.payload.as_object_mut() {
obj.insert(
"coalesce_reschedule_count".to_string(),
json!(reschedule_count + 1),
);
}
self.repository.update_task_guarded(&updated).await?;
Ok(None)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::models::{Task, TaskType};
use crate::domain::repositories::task_repository::{RepositoryError, TaskQueryParams};
use crate::engines::types::ScrapeOptions;
use async_trait::async_trait;
use serde_json::json;
use std::collections::HashSet;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Mutex;
struct CountingTaskRepo {
update_calls: AtomicU32,
mark_completed_calls: AtomicU32,
mark_failed_calls: AtomicU32,
captured_updates: Mutex<Vec<Task>>,
}
impl CountingTaskRepo {
fn new() -> Self {
Self {
update_calls: AtomicU32::new(0),
mark_completed_calls: AtomicU32::new(0),
mark_failed_calls: AtomicU32::new(0),
captured_updates: Mutex::new(Vec::new()),
}
}
}
#[async_trait]
impl TaskRepository for CountingTaskRepo {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
self.update_calls.fetch_add(1, Ordering::SeqCst);
self.captured_updates.lock().unwrap().push(task.clone());
Ok(task.clone())
}
async fn update_task_guarded(&self, task: &Task) -> Result<u64, RepositoryError> {
self.update_calls.fetch_add(1, Ordering::SeqCst);
self.captured_updates.lock().unwrap().push(task.clone());
Ok(1)
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
self.mark_completed_calls.fetch_add(1, Ordering::SeqCst);
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
self.mark_failed_calls.fetch_add(1, Ordering::SeqCst);
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
struct ConfigurableResultRepo {
find_calls: AtomicU32,
return_some: Mutex<bool>,
last_query_task_id: Mutex<Option<crate::domain::id::Id>>,
first_query_task_id: Mutex<Option<crate::domain::id::Id>>,
save_calls: AtomicU32,
last_saved_task_id: Mutex<Option<crate::domain::id::Id>>,
some_only_for: Mutex<Option<crate::domain::id::Id>>,
}
impl ConfigurableResultRepo {
fn new(return_some: bool) -> Self {
Self {
find_calls: AtomicU32::new(0),
return_some: Mutex::new(return_some),
last_query_task_id: Mutex::new(None),
first_query_task_id: Mutex::new(None),
save_calls: AtomicU32::new(0),
last_saved_task_id: Mutex::new(None),
some_only_for: Mutex::new(None),
}
}
fn new_with_result_only_for(leader_id: crate::domain::id::Id) -> Self {
Self {
find_calls: AtomicU32::new(0),
return_some: Mutex::new(true),
last_query_task_id: Mutex::new(None),
first_query_task_id: Mutex::new(None),
save_calls: AtomicU32::new(0),
last_saved_task_id: Mutex::new(None),
some_only_for: Mutex::new(Some(leader_id)),
}
}
}
#[async_trait]
impl ScrapeResultRepository for ConfigurableResultRepo {
async fn save(&self, result: crate::domain::models::ScrapeResult) -> Result<()> {
self.save_calls.fetch_add(1, Ordering::SeqCst);
*self.last_saved_task_id.lock().unwrap() = Some(result.task_id);
Ok(())
}
async fn find_by_task_id(
&self,
task_id: crate::domain::id::Id,
) -> Result<Option<crate::domain::models::ScrapeResult>> {
self.find_calls.fetch_add(1, Ordering::SeqCst);
let mut first = self.first_query_task_id.lock().unwrap();
if first.is_none() {
*first = Some(task_id);
}
drop(first);
*self.last_query_task_id.lock().unwrap() = Some(task_id);
let hit = {
let only_for = self.some_only_for.lock().unwrap();
match &*only_for {
Some(lid) => task_id == *lid,
None => *self.return_some.lock().unwrap(),
}
};
if hit {
Ok(Some(crate::domain::models::ScrapeResult {
id: crate::domain::id::Id::generate(),
task_id: crate::domain::id::Id::generate(),
url: String::new(),
status_code: 200,
content: String::new(),
content_type: "text/html".to_string(),
headers: serde_json::json!({}),
meta_data: serde_json::json!({}),
screenshot: None,
response_time_ms: 0,
created_at: Utc::now(),
}))
} else {
Ok(None)
}
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> Result<Vec<crate::domain::models::ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(&self, _team_id: crate::domain::id::Id) -> Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
fn make_task() -> Task {
Task::new(
crate::domain::id::Id::generate(),
TaskType::Scrape,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
json!({}),
)
}
fn key_for(task: &Task) -> CoalesceKey {
CoalesceKey::new(task.team_id, &task.url, &ScrapeOptions::default())
}
fn finish_leader_success(result: CoalesceResult) {
match result {
CoalesceResult::Proceed(mut guard) => {
guard.mark_success();
drop(guard);
}
CoalesceResult::Wait(..) => panic!("test leader must acquire coalesce slot"),
}
}
fn finish_leader_failure(result: CoalesceResult) {
match result {
CoalesceResult::Proceed(guard) => drop(guard),
CoalesceResult::Wait(..) => panic!("test leader must acquire coalesce slot"),
}
}
#[tokio::test]
async fn try_coalesce_proceed_returns_some_guard() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let result_repo = Arc::new(ConfigurableResultRepo::new(false));
let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let task = make_task();
let result = coord.try_coalesce(&key_for(&task), &task, worker_id).await;
assert!(result.is_ok(), "proceed path should succeed");
let guard = result.unwrap();
assert!(guard.is_some(), "first call should return Some(guard)");
assert_eq!(task_repo.update_calls.load(Ordering::SeqCst), 0);
assert_eq!(task_repo.mark_completed_calls.load(Ordering::SeqCst), 0);
assert_eq!(result_repo.find_calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn try_coalesce_wait_reschedules_when_result_missing() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let result_repo = Arc::new(ConfigurableResultRepo::new(false)); let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let task = make_task();
let key = key_for(&task);
let first_guard = coalescer.try_start(&key, crate::domain::id::Id::generate());
let task_clone = task.clone();
let coord_clone = coord.clone();
let key_clone = key.clone();
let handle = tokio::spawn(async move {
coord_clone
.try_coalesce(&key_clone, &task_clone, worker_id)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
finish_leader_success(first_guard);
let result = handle.await.expect("task panicked");
assert!(result.is_ok(), "should not error");
let guard = result.unwrap();
assert!(guard.is_none(), "should return None (rescheduled)");
assert_eq!(
task_repo.update_calls.load(Ordering::SeqCst),
1,
"update should be called once for reschedule"
);
let captured = task_repo.captured_updates.lock().unwrap();
let updated = captured.last().expect("captured update");
assert_eq!(updated.status, TaskStatus::Queued);
assert!(
updated.scheduled_at.is_some(),
"scheduled_at should be set for reschedule"
);
assert_eq!(
task_repo.mark_completed_calls.load(Ordering::SeqCst),
0,
"mark_completed should not be called when result missing"
);
}
#[tokio::test]
async fn try_coalesce_failed_leader_reschedules_without_result_lookup() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let result_repo = Arc::new(ConfigurableResultRepo::new(true));
let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let task = make_task();
let key = key_for(&task);
let leader_guard = coalescer.try_start(&key, crate::domain::id::Id::generate());
let task_clone = task.clone();
let coord_clone = coord.clone();
let key_clone = key.clone();
let handle = tokio::spawn(async move {
coord_clone
.try_coalesce(&key_clone, &task_clone, worker_id)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
finish_leader_failure(leader_guard);
let result = handle.await.expect("task panicked");
assert!(result.is_ok());
assert!(
result.unwrap().is_none(),
"failed leader should reschedule waiter (None)"
);
assert_eq!(
result_repo.find_calls.load(Ordering::SeqCst),
0,
"failed leader must NOT trigger result lookup"
);
assert_eq!(
task_repo.mark_completed_calls.load(Ordering::SeqCst),
0,
"failed leader must NOT mark waiter completed"
);
assert_eq!(
task_repo.update_calls.load(Ordering::SeqCst),
1,
"failed leader should reschedule waiter via guarded update"
);
}
#[tokio::test]
async fn try_coalesce_wait_completes_when_result_found() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let result_repo = Arc::new(ConfigurableResultRepo::new(true)); let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let task = make_task();
let key = key_for(&task);
let first_guard = coalescer.try_start(&key, crate::domain::id::Id::generate());
let task_clone = task.clone();
let coord_clone = coord.clone();
let key_clone = key.clone();
let handle = tokio::spawn(async move {
coord_clone
.try_coalesce(&key_clone, &task_clone, worker_id)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
finish_leader_success(first_guard);
let result = handle.await.expect("task panicked");
assert!(result.is_ok(), "should not error");
let guard = result.unwrap();
assert!(guard.is_none(), "should return None (completed)");
assert_eq!(
task_repo.mark_completed_calls.load(Ordering::SeqCst),
1,
"mark_completed should be called when result found"
);
assert_eq!(
task_repo.update_calls.load(Ordering::SeqCst),
0,
"update should not be called when result found"
);
}
#[tokio::test]
async fn try_coalesce_wait_does_not_call_find_while_waiting() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let result_repo = Arc::new(ConfigurableResultRepo::new(false));
let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let task = make_task();
let key = key_for(&task);
let _first_guard = coalescer.try_start(&key, crate::domain::id::Id::generate());
let task_clone = task.clone();
let coord_clone = coord.clone();
let key_clone = key.clone();
let handle = tokio::spawn(async move {
coord_clone
.try_coalesce(&key_clone, &task_clone, worker_id)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert_eq!(
result_repo.find_calls.load(Ordering::SeqCst),
0,
"find_by_task_id should not be called while waiting for broadcast"
);
handle.abort();
}
#[tokio::test]
async fn try_coalesce_max_reschedule_exceeded_marks_failed() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let result_repo = Arc::new(ConfigurableResultRepo::new(false));
let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let mut task = make_task();
task.payload = json!({"coalesce_reschedule_count": 3});
let key = key_for(&task);
let first_guard = coalescer.try_start(&key, crate::domain::id::Id::generate());
let task_clone = task.clone();
let coord_clone = coord.clone();
let key_clone = key.clone();
let handle = tokio::spawn(async move {
coord_clone
.try_coalesce(&key_clone, &task_clone, worker_id)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
finish_leader_success(first_guard);
let result = handle.await.expect("task panicked");
assert!(result.is_ok());
let guard = result.unwrap();
assert!(
guard.is_none(),
"max reschedule exceeded should return None"
);
assert_eq!(
task_repo.mark_failed_calls.load(Ordering::SeqCst),
0,
"仅 mark_failed 会丢失失败原因,超限路径不得使用"
);
assert_eq!(
task_repo.update_calls.load(Ordering::SeqCst),
1,
"超限路径应恰好执行一次守卫式更新"
);
let captured = task_repo.captured_updates.lock().unwrap();
let failed = captured.last().expect("应捕获失败快照");
assert_eq!(failed.status, TaskStatus::Failed);
let reason = failed
.payload
.get("error")
.and_then(|v| v.as_str())
.unwrap_or_default();
assert!(
reason.contains("reschedule limit"),
"失败原因必须落库,实际: {reason}"
);
}
#[tokio::test]
async fn try_coalesce_wait_find_by_task_id_error_propagates() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let coalescer = Arc::new(RequestCoalescer::new());
struct ErrorResultRepo;
#[async_trait]
impl ScrapeResultRepository for ErrorResultRepo {
async fn save(&self, _result: crate::domain::models::ScrapeResult) -> Result<()> {
Ok(())
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> Result<Option<crate::domain::models::ScrapeResult>> {
Err(anyhow::anyhow!("db connection lost"))
}
async fn find_by_task_ids(
&self,
_task_ids: &[crate::domain::id::Id],
) -> Result<Vec<crate::domain::models::ScrapeResult>> {
Ok(vec![])
}
async fn get_team_avg_response_time(
&self,
_team_id: crate::domain::id::Id,
) -> Result<f64> {
Ok(0.0)
}
async fn cleanup_expired(&self, _retention_days: i64) -> anyhow::Result<u64> {
Ok(0)
}
}
let result_repo = Arc::new(ErrorResultRepo);
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let task = make_task();
let key = key_for(&task);
let first_guard = coalescer.try_start(&key, crate::domain::id::Id::generate());
let task_clone = task.clone();
let coord_clone = coord.clone();
let key_clone = key.clone();
let handle = tokio::spawn(async move {
coord_clone
.try_coalesce(&key_clone, &task_clone, worker_id)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
finish_leader_success(first_guard);
let result = handle.await.expect("task panicked");
assert!(result.is_err(), "find_by_task_id error should propagate");
}
#[test]
fn test_coalesce_coordinator_new() {
let task_repo = Arc::new(CountingTaskRepo::new());
let result_repo = Arc::new(ConfigurableResultRepo::new(false));
let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let _cloned = coord.clone();
}
#[tokio::test]
async fn try_coalesce_closed_channel_reschedules_not_failed() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let result_repo = Arc::new(ConfigurableResultRepo::new(true));
let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let task = make_task();
let key = key_for(&task);
let _leader_guard = coalescer.try_start(&key, crate::domain::id::Id::generate());
let task_clone = task.clone();
let coord_clone = coord.clone();
let key_clone = key.clone();
let handle = tokio::spawn(async move {
coord_clone
.try_coalesce(&key_clone, &task_clone, worker_id)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
coalescer.drop_entry_silently(&key);
let result = handle.await.expect("task panicked");
assert!(result.is_ok(), "closed channel should not error");
assert!(
result.unwrap().is_none(),
"closed channel should reschedule self (None)"
);
assert_eq!(
task_repo.mark_failed_calls.load(Ordering::SeqCst),
0,
"closed channel must NOT mark waiter task failed"
);
assert_eq!(
task_repo.update_calls.load(Ordering::SeqCst),
1,
"closed channel should reschedule self via guarded update"
);
assert_eq!(
result_repo.find_calls.load(Ordering::SeqCst),
0,
"closed channel must not query result_repo"
);
}
#[tokio::test]
async fn try_coalesce_purged_signal_skips_result_lookup() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let result_repo = Arc::new(ConfigurableResultRepo::new(true));
let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let task = make_task();
let key = key_for(&task);
let _leader_guard = coalescer.try_start(&key, crate::domain::id::Id::generate());
let task_clone = task.clone();
let coord_clone = coord.clone();
let key_clone = key.clone();
let handle = tokio::spawn(async move {
coord_clone
.try_coalesce(&key_clone, &task_clone, worker_id)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
coalescer.purge_key(&key);
let result = handle.await.expect("task panicked");
assert!(result.is_ok());
assert!(
result.unwrap().is_none(),
"purged signal should reschedule self (None)"
);
assert_eq!(
result_repo.find_calls.load(Ordering::SeqCst),
0,
"Purged must NOT trigger result lookup"
);
assert_eq!(
task_repo.mark_completed_calls.load(Ordering::SeqCst),
0,
"Purged must NOT mark completed"
);
assert_eq!(
task_repo.mark_failed_calls.load(Ordering::SeqCst),
0,
"Purged must NOT mark waiter failed"
);
assert_eq!(
task_repo.update_calls.load(Ordering::SeqCst),
1,
"Purged should reschedule self via guarded update"
);
}
#[tokio::test]
async fn try_coalesce_queries_result_by_leader_task_id() {
let worker_id = crate::domain::id::Id::generate();
let task_repo = Arc::new(CountingTaskRepo::new());
let leader_id = crate::domain::id::Id::generate();
let result_repo = Arc::new(ConfigurableResultRepo::new_with_result_only_for(leader_id));
let coalescer = Arc::new(RequestCoalescer::new());
let coord =
CoalesceCoordinator::new(task_repo.clone(), result_repo.clone(), coalescer.clone());
let waiter_task = make_task();
assert_ne!(
leader_id, waiter_task.id,
"test precondition: leader and waiter task ids differ"
);
let key = key_for(&waiter_task);
let leader_guard = coalescer.try_start(&key, leader_id);
let task_clone = waiter_task.clone();
let coord_clone = coord.clone();
let key_clone = key.clone();
let handle = tokio::spawn(async move {
coord_clone
.try_coalesce(&key_clone, &task_clone, worker_id)
.await
});
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
finish_leader_success(leader_guard);
let result = handle.await.expect("task panicked");
assert!(result.is_ok());
assert!(result.unwrap().is_none());
let first_queried = *result_repo.first_query_task_id.lock().unwrap();
assert_eq!(
first_queried,
Some(leader_id),
"waiter must query copy source by leader task_id"
);
assert_eq!(
result_repo.save_calls.load(Ordering::SeqCst),
1,
"leader result must be copied once under the waiter task"
);
assert_eq!(
*result_repo.last_saved_task_id.lock().unwrap(),
Some(waiter_task.id),
"copied result must be re-attributed to the waiter task_id"
);
assert_eq!(
*result_repo.last_query_task_id.lock().unwrap(),
Some(waiter_task.id),
"copy must be idempotent-guarded by a waiter-side result lookup"
);
assert_eq!(task_repo.mark_completed_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn worker_same_url_different_team_does_not_coalesce() {
let worker_id = crate::domain::id::Id::generate();
let coalescer = Arc::new(RequestCoalescer::new());
let coord = CoalesceCoordinator::new(
Arc::new(CountingTaskRepo::new()),
Arc::new(ConfigurableResultRepo::new(false)),
coalescer.clone(),
);
let task_a = make_task();
let mut task_b = make_task();
task_b.team_id = crate::domain::id::Id::generate();
task_b.url = task_a.url.clone();
assert_ne!(task_a.team_id, task_b.team_id);
let key_a = key_for(&task_a);
let key_b = key_for(&task_b);
let (result_a, result_b) = tokio::join!(
coord.try_coalesce(&key_a, &task_a, worker_id),
coord.try_coalesce(&key_b, &task_b, worker_id),
);
assert!(result_a.is_ok(), "tenant A should proceed");
assert!(result_b.is_ok(), "tenant B should proceed");
let (guard_a, guard_b) = (result_a.unwrap(), result_b.unwrap());
assert!(guard_a.is_some(), "tenant A must acquire coalesce slot");
assert!(
guard_b.is_some(),
"tenant B must not wait on tenant A's in-flight entry"
);
assert_eq!(coalescer.in_flight_count(), 2);
}
#[tokio::test]
async fn worker_same_tenant_different_headers_do_not_coalesce() {
let worker_id = crate::domain::id::Id::generate();
let coalescer = Arc::new(RequestCoalescer::new());
let coord = CoalesceCoordinator::new(
Arc::new(CountingTaskRepo::new()),
Arc::new(ConfigurableResultRepo::new(false)),
coalescer.clone(),
);
let task = make_task();
let mut options_a = ScrapeOptions::default();
options_a
.headers
.insert("X-Trace".to_string(), "a".to_string());
let mut options_b = ScrapeOptions::default();
options_b
.headers
.insert("X-Trace".to_string(), "b".to_string());
let key_a = CoalesceKey::new(task.team_id, &task.url, &options_a);
let key_b = CoalesceKey::new(task.team_id, &task.url, &options_b);
let (result_a, result_b) = tokio::join!(
coord.try_coalesce(&key_a, &task, worker_id),
coord.try_coalesce(&key_b, &task, worker_id),
);
assert!(result_a.is_ok());
assert!(result_b.is_ok());
let (guard_a, guard_b) = (result_a.unwrap(), result_b.unwrap());
assert!(guard_a.is_some(), "header set A must proceed");
assert!(
guard_b.is_some(),
"header set B must not coalesce with header set A"
);
assert_eq!(coalescer.in_flight_count(), 2);
}
#[cfg(debug_assertions)]
#[tokio::test]
#[should_panic(expected = "coalesce key tenant must match task tenant")]
async fn try_coalesce_tenant_mismatch_panics_in_debug() {
let worker_id = crate::domain::id::Id::generate();
let coalescer = Arc::new(RequestCoalescer::new());
let coord = CoalesceCoordinator::new(
Arc::new(CountingTaskRepo::new()),
Arc::new(ConfigurableResultRepo::new(false)),
coalescer,
);
let task = make_task();
let other_team = if task.team_id == crate::domain::id::Id::from_u128(0) {
crate::domain::id::Id::from_u128(1)
} else {
crate::domain::id::Id::from_u128(0)
};
let key = CoalesceKey::new(other_team, &task.url, &ScrapeOptions::default());
let _ = coord.try_coalesce(&key, &task, worker_id).await;
}
}