use std::collections::BTreeMap;
use crate::batching::models::Partition;
use crate::config::models::LoadErrorPolicy;
use crate::delivery::constants::{CHECK_RETRY, HISTORY_MARGIN, RECEIVED_BEFORE_LOOKBACK};
use crate::delivery::models::{
CheckTiming, FileState, PendingBatch, PendingFile, PendingLoads, Settled,
};
use crate::stage_api::models::{FileLoad, LoadStatus};
impl PendingLoads {
pub(crate) fn new(policy: LoadErrorPolicy, timing: CheckTiming) -> Self {
Self {
batches: Default::default(),
policy,
timing,
}
}
pub(crate) fn is_empty(&self) -> bool {
self.batches.is_empty()
}
pub(crate) fn file_count(&self) -> usize {
self.files().count()
}
pub(crate) fn oldest_sent_ms(&self) -> Option<i64> {
self.files().map(|file| file.first_sent_ms).min()
}
pub(crate) fn add(
&mut self,
keys: &[String],
skipped: &[String],
offsets: BTreeMap<Partition, i64>,
now: i64,
) {
let check_after = self.timing.check_after.as_millis() as i64;
let files: Vec<PendingFile> = keys
.iter()
.map(|key| {
let received_before = is_listed(skipped, key);
PendingFile {
key: key.clone(),
first_sent_ms: now,
resent: false,
received_before,
next_check_ms: match received_before {
true => now,
false => now + check_after,
},
last_check_ms: None,
failed_checks: 0,
warned_no_history: false,
state: FileState::Waiting,
}
})
.collect();
self.batches.push_back(PendingBatch { files, offsets });
}
pub(crate) fn apply(&mut self, loads: &[FileLoad]) {
for load in loads {
for file in self.files_mut() {
if !same_path(&load.path, &file.key) || !is_unsettled(file) {
continue;
}
file.state = match load.status {
LoadStatus::Loaded => FileState::Loaded,
LoadStatus::InProgress => FileState::InProgress,
LoadStatus::Failed | LoadStatus::PartiallyLoaded => {
FileState::Failed(failure(load))
}
};
}
}
}
pub(crate) fn overdue(&self, now: i64) -> Vec<String> {
self.files()
.filter(|file| is_unsettled(file) && now >= file.next_check_ms)
.map(|file| file.key.clone())
.collect()
}
pub(crate) fn mark_all_due(&mut self, now: i64) {
let recent = self.retry_ms();
for file in self.files_mut().filter(|file| is_unsettled(file)) {
if file.last_check_ms.is_none_or(|last| now - last >= recent) {
file.next_check_ms = file.next_check_ms.min(now);
}
}
}
pub(crate) fn checked(&mut self, keys: &[String], now: i64) {
let recheck = self.timing.recheck_every.as_millis() as i64;
for file in self.files_mut().filter(|file| keys.contains(&file.key)) {
file.last_check_ms = Some(now);
file.next_check_ms = now + recheck;
file.failed_checks = 0;
}
}
pub(crate) fn check_failed(&mut self, keys: &[String], now: i64) {
let retry = self.retry_ms();
let recheck = self.timing.recheck_every.as_millis() as i64;
for file in self.files_mut().filter(|file| keys.contains(&file.key)) {
let factor = 1i64 << file.failed_checks.min(16);
file.failed_checks += 1;
file.next_check_ms = now + retry.saturating_mul(factor).min(recheck);
}
}
pub(crate) fn history_start(&self, keys: &[String]) -> Option<i64> {
self.files()
.filter(|file| keys.contains(&file.key))
.map(|file| {
let lookback = match file.received_before {
true => RECEIVED_BEFORE_LOOKBACK,
false => HISTORY_MARGIN,
};
file.first_sent_ms - lookback.as_millis() as i64
})
.min()
}
pub(crate) fn any_unsettled(&self, keys: &[String]) -> bool {
self.files()
.any(|file| keys.contains(&file.key) && is_unsettled(file))
}
pub(crate) fn defer(&mut self, keys: &[String], not_before: i64) {
for file in self.files_mut().filter(|file| keys.contains(&file.key)) {
file.next_check_ms = file.next_check_ms.max(not_before);
}
}
pub(crate) fn waiting(&self) -> Vec<String> {
self.files()
.filter(|file| is_unsettled(file))
.map(|file| file.key.clone())
.collect()
}
pub(crate) fn take_unwarned(&mut self, keys: &[String]) -> Vec<String> {
let mut found: Vec<String> = Vec::new();
for file in self.files_mut().filter(|file| keys.contains(&file.key)) {
if !file.warned_no_history {
file.warned_no_history = true;
found.push(file.key.clone());
}
}
found
}
pub(crate) fn take_unwarned_received_before(&mut self, keys: &[String]) -> Vec<String> {
let mut found: Vec<String> = Vec::new();
for file in self.files_mut().filter(|file| keys.contains(&file.key)) {
if file.received_before && !file.warned_no_history {
file.warned_no_history = true;
found.push(file.key.clone());
}
}
found
}
pub(crate) fn is_acknowledged(&self, key: &str) -> bool {
self.files()
.any(|file| file.key == key && (file.resent || file.received_before))
}
pub(crate) fn resend(&mut self, key: &str, skipped: bool) {
for file in self.files_mut().filter(|file| file.key == key) {
file.resent = true;
file.received_before |= skipped;
}
}
pub(crate) fn fail(&mut self, key: &str, reason: &str) {
for file in self.files_mut().filter(|file| file.key == key) {
file.state = FileState::Failed(format!("{}: {reason}", file.key));
}
}
pub(crate) fn settle(&mut self) -> Settled {
let mut settled = Settled::default();
let policy = self.policy;
for file in self.files_mut() {
let FileState::Failed(reason) = &file.state else {
continue;
};
match policy {
LoadErrorPolicy::Halt => {
settled.halted.get_or_insert_with(|| reason.clone());
}
LoadErrorPolicy::Skip => {
settled.skipped.push((file.key.clone(), reason.clone()));
file.state = FileState::Skipped;
}
}
}
while let Some(batch) = self.batches.front() {
if !batch.files.iter().all(is_done) {
break;
}
if let Some(batch) = self.batches.pop_front() {
for (partition, offset) in batch.offsets {
let entry = settled.confirmed.entry(partition).or_insert(offset);
*entry = (*entry).max(offset);
}
}
}
settled
}
pub(crate) fn forget(&mut self, partitions: &[Partition]) {
for batch in &mut self.batches {
batch
.offsets
.retain(|partition, _| !partitions.contains(partition));
}
self.batches.retain(|batch| !batch.offsets.is_empty());
}
pub(crate) fn clear(&mut self) {
self.batches.clear();
}
fn retry_ms(&self) -> i64 {
CHECK_RETRY.min(self.timing.check_after).as_millis() as i64
}
fn files(&self) -> impl Iterator<Item = &PendingFile> {
self.batches.iter().flat_map(|batch| batch.files.iter())
}
fn files_mut(&mut self) -> impl Iterator<Item = &mut PendingFile> {
self.batches
.iter_mut()
.flat_map(|batch| batch.files.iter_mut())
}
}
fn is_listed(paths: &[String], key: &str) -> bool {
paths.iter().any(|path| same_path(path, key))
}
fn is_done(file: &PendingFile) -> bool {
matches!(file.state, FileState::Loaded | FileState::Skipped)
}
fn is_unsettled(file: &PendingFile) -> bool {
matches!(file.state, FileState::Waiting | FileState::InProgress)
}
pub(crate) fn same_path(reported: &str, key: &str) -> bool {
reported.trim_start_matches('/') == key.trim_start_matches('/')
}
fn failure(load: &FileLoad) -> String {
let status = match load.status {
LoadStatus::PartiallyLoaded => format!(
"partially loaded ({} of {} rows)",
load.rows_inserted, load.rows_parsed
),
_ => "load failed".to_owned(),
};
match &load.first_error {
Some(error) => format!("{}: {status}: {error}", load.path),
None => format!("{}: {status}", load.path),
}
}