use dashmap::DashMap;
use serde::{Deserialize, Serialize};
use std::io::Write;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use super::node_identity::now_secs;
use super::wal::{Wal, WalStatus};
pub const ANONYMOUS_USER: &str = "anonymous";
const MAX_FLUSH_ATTEMPTS: u32 = 5;
const MAX_FLUSH_BATCH: usize = 500;
const MAX_FLUSH_BYTES: usize = 512 * 1024;
fn fill_chunk<T>(
src: &mut impl Iterator<Item = (T, usize)>,
carry: &mut Option<(T, usize)>,
) -> Vec<T> {
let mut out = Vec::new();
let mut bytes = 2usize; while let Some((item, size)) = carry.take().or_else(|| src.next()) {
let add = size + 1; if !out.is_empty() && (out.len() >= MAX_FLUSH_BATCH || bytes + add > MAX_FLUSH_BYTES) {
*carry = Some((item, size));
break;
}
bytes += add;
out.push(item);
}
out
}
fn retire_delivered(wal: &Wal, tx: &BufferedTransaction) {
let _ = wal.update_status(
&tx.request_id,
WalStatus::Committed,
Some(tx.amount),
Some(tx.duration_ms),
);
}
#[cfg(test)]
mod settlement_liveness_tests {
use super::*;
#[test]
fn a_fresh_buffer_is_not_already_stale() {
let buf = TransactionBuffer::new();
assert!(buf.seconds_since_flush() < 5);
}
#[test]
fn an_idle_broker_still_proves_the_flush_path_runs() {
let buf = TransactionBuffer::new();
let path = std::env::temp_dir().join(format!(
"zc-liveness-{}.wal",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_file(&path);
let wal = Wal::open(&path).expect("open test WAL");
buf.last_flush_at.store(0, Ordering::Relaxed);
assert!(buf.seconds_since_flush() > 1000, "fixture must look stale");
assert_eq!(buf.pending_count(), 0);
buf.flush_to_api("http://127.0.0.1:1", "k", &wal);
assert!(
buf.seconds_since_flush() < 5,
"an empty flush must still stamp liveness"
);
let _ = std::fs::remove_file(&path);
}
#[test]
fn pending_work_is_visible_before_it_is_ever_quarantined() {
let buf = TransactionBuffer::new();
buf.push_transaction(BufferedTransaction {
request_id: "r1".into(),
user_id: "u1".into(),
tx_type: "commit".into(),
amount: 0.003,
balance_after: 1.0,
worker_id: "w1".into(),
duration_ms: 1.0,
source_node: None,
worker_name: None,
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
});
assert_eq!(buf.pending_count(), 1);
assert_eq!(buf.total_quarantined(), 0, "nothing failed — nothing ran");
}
#[test]
fn metadata_for_a_transaction_that_was_never_queued_is_not_kept() {
let buf = TransactionBuffer::new();
buf.attach_metadata("never-queued", serde_json::json!({"tasks": []}));
assert!(buf.metadata.is_empty());
buf.push_transaction(BufferedTransaction {
request_id: "r1".into(),
user_id: "u1".into(),
tx_type: "commit".into(),
amount: 0.003,
balance_after: 1.0,
worker_id: "w1".into(),
duration_ms: 1.0,
source_node: None,
worker_name: None,
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
});
buf.attach_metadata("r1", serde_json::json!({"tasks": []}));
assert_eq!(buf.metadata.len(), 1);
}
}
const MAX_RECENT_QUARANTINED_PER_ACCOUNT: usize = 50;
pub(crate) const MAX_QUARANTINED_ACCOUNTS: usize = 1024;
const MAX_LISTED_PENDING_PER_ACCOUNT: usize = 100;
fn visible_request_id(tx: &BufferedTransaction) -> Option<String> {
if tx.tx_type == "earning" || tx.request_id.starts_with("earn-") {
None
} else {
Some(tx.request_id.clone())
}
}
#[derive(Debug, Default)]
struct QuarantineRecords {
by_account: std::collections::HashMap<String, AccountQuarantine>,
seq: u64,
}
#[derive(Debug, Default)]
struct AccountQuarantine {
count: u64,
last_seq: u64,
rows: std::collections::VecDeque<AccountQuarantinedRow>,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
pub struct AccountPendingRow {
#[serde(skip_serializing_if = "Option::is_none")]
pub request_id: Option<String>,
pub tx_type: String,
pub amount: f64,
pub attempts: u32,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
pub struct AccountQuarantinedRow {
#[serde(skip_serializing_if = "Option::is_none")]
pub request_id: Option<String>,
pub tx_type: String,
pub amount: f64,
pub quarantined_at: String,
}
#[derive(Debug, Clone, Serialize, PartialEq)]
pub struct AccountSettlement {
pub account: String,
pub settlement_pending: usize,
pub settlement_quarantined: u64,
pub quarantine_incomplete: bool,
pub pending: Vec<AccountPendingRow>,
pub quarantined: Vec<AccountQuarantinedRow>,
}
#[cfg(test)]
const HUB_MAX_BODY_BYTES: usize = 1024 * 1024;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BufferedTransaction {
pub request_id: String,
pub user_id: String,
pub tx_type: String,
pub amount: f64,
pub balance_after: f64,
pub worker_id: String,
pub duration_ms: f64,
pub source_node: Option<String>,
pub worker_name: Option<String>,
pub worker_uri: Option<String>,
pub price_per_hour: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub voucher_nonce: Option<String>,
}
pub struct TransactionBuffer {
pending: DashMap<String, BufferedTransaction>,
balance_snapshots: DashMap<String, f64>,
total_flushed: AtomicU64,
total_quarantined: AtomicU64,
total_retired: AtomicU64,
total_dropped: AtomicU64,
last_flush_at: AtomicU64,
attempts: DashMap<String, u32>,
metadata: DashMap<String, serde_json::Value>,
quarantined_by_account: std::sync::Mutex<QuarantineRecords>,
quarantine_records_evicted: AtomicU64,
anonymous_warned: AtomicBool,
}
pub(crate) fn dashboard_type(tx_type: &str) -> &str {
match tx_type {
"commit" | "cancel" => "job_execution",
other => other,
}
}
fn job_name_for(tx: &BufferedTransaction) -> String {
match tx.tx_type.as_str() {
"cancel" => format!(
"Cancelled Job ({})",
tx.worker_name.as_deref().unwrap_or(&tx.worker_id)
),
_ => format!(
"Compute Job ({})",
tx.worker_name.as_deref().unwrap_or(&tx.worker_id)
),
}
}
const PERMANENT_ITEM_ERRORS: &[&[&str]] = &[
&["User not found"],
&["(expired)"],
&["(over_budget)"],
&["(wrong_requester)"],
&[
"a user key may post ",
" only for a recorded job (request_id earn-<rid> or <rid>-earn-refund) (#403)",
],
&[
"the charge for ",
" names no worker, so no provider can claim it (#403)",
],
&["a refund may go only to the account that was charged (#403)"],
&[
"earnings and refunds for ",
" would exceed its charge ",
" (#403)",
],
&["transaction_type '", "' requires a service credential"],
];
fn floor_to_hub_precision(amount: f64) -> f64 {
if !amount.is_finite() || amount <= 0.0 {
return amount;
}
((amount * 1e6) + 1e-6).floor() / 1e6
}
fn is_permanent_item_error(error: &str) -> bool {
PERMANENT_ITEM_ERRORS.iter().any(|fragments| {
let mut rest = error;
fragments.iter().all(|f| match rest.find(f) {
Some(at) => {
rest = &rest[at + f.len()..];
true
}
None => false,
})
})
}
const MAX_SPLIT_REQUESTS_PER_FLUSH: usize = 64;
fn summarize_item_errors(errors: &serde_json::Value) -> String {
match errors.as_array() {
Some(errors) if !errors.is_empty() => {
let joined: Vec<String> = errors
.iter()
.take(3)
.map(|e| {
format!(
"{}: {}",
e["job_name"].as_str().unwrap_or("?"),
e["error"].as_str().unwrap_or("unknown")
)
})
.collect();
let mut summary = format!(
"hub reported {} item failure(s): {}",
errors.len(),
joined.join("; ")
);
if errors.len() > 3 {
summary.push_str(&format!(" … and {} more", errors.len() - 3));
}
summary
}
_ => "hub reported this item as failed (no error detail provided)".to_string(),
}
}
impl TransactionBuffer {
pub fn new() -> Self {
Self {
pending: DashMap::new(),
balance_snapshots: DashMap::new(),
total_flushed: AtomicU64::new(0),
total_quarantined: AtomicU64::new(0),
total_retired: AtomicU64::new(0),
total_dropped: AtomicU64::new(0),
last_flush_at: AtomicU64::new(now_secs()),
attempts: DashMap::new(),
metadata: DashMap::new(),
quarantined_by_account: std::sync::Mutex::new(QuarantineRecords::default()),
quarantine_records_evicted: AtomicU64::new(0),
anonymous_warned: AtomicBool::new(false),
}
}
pub fn push_transaction(&self, tx: BufferedTransaction) {
if tx.user_id == ANONYMOUS_USER {
if !self.anonymous_warned.swap(true, Ordering::Relaxed) {
eprintln!(
" [FLUSH] Skipping hub sync for anonymous-user transaction {} \
(local execution; nothing to settle)",
tx.request_id
);
}
return;
}
self.pending.insert(tx.request_id.clone(), tx);
}
pub fn attach_metadata(&self, request_id: &str, metadata: serde_json::Value) {
if self.pending.contains_key(request_id) {
self.metadata.insert(request_id.to_string(), metadata);
}
}
pub fn snapshot_balance(&self, user_id: &str, balance: f64) {
self.balance_snapshots.insert(user_id.to_string(), balance);
}
pub fn pending_count(&self) -> usize {
self.pending.len()
}
pub fn total_flushed(&self) -> u64 {
self.total_flushed.load(Ordering::Relaxed)
}
pub fn seconds_since_flush(&self) -> u64 {
now_secs().saturating_sub(self.last_flush_at.load(Ordering::Relaxed))
}
pub fn total_quarantined(&self) -> u64 {
self.total_quarantined.load(Ordering::Relaxed)
}
pub fn total_retired(&self) -> u64 {
self.total_retired.load(Ordering::Relaxed)
}
pub fn total_dropped(&self) -> u64 {
self.total_dropped.load(Ordering::Relaxed)
}
pub fn account_settlement(&self, account: &str) -> AccountSettlement {
let settlement_pending = self
.pending
.iter()
.filter(|e| e.value().user_id == account)
.count();
let pending: Vec<AccountPendingRow> = self
.pending
.iter()
.filter(|e| e.value().user_id == account)
.take(MAX_LISTED_PENDING_PER_ACCOUNT)
.map(|e| {
let tx = e.value();
AccountPendingRow {
request_id: visible_request_id(tx),
tx_type: tx.tx_type.clone(),
amount: tx.amount,
attempts: self.attempts.get(&tx.request_id).map(|a| *a).unwrap_or(0),
}
})
.collect();
let (settlement_quarantined, quarantined) = self
.quarantine_records()
.by_account
.get(account)
.map(|e| (e.count, e.rows.iter().cloned().collect()))
.unwrap_or((0, Vec::new()));
AccountSettlement {
account: account.to_string(),
settlement_pending,
settlement_quarantined,
quarantine_incomplete: self.quarantine_records_evicted() > 0,
pending,
quarantined,
}
}
pub fn quarantine_records_evicted(&self) -> u64 {
self.quarantine_records_evicted.load(Ordering::Relaxed)
}
fn quarantine_records(&self) -> std::sync::MutexGuard<'_, QuarantineRecords> {
self.quarantined_by_account
.lock()
.unwrap_or_else(|e| e.into_inner())
}
fn note_quarantined(&self, tx: &BufferedTransaction, quarantined_at: &str) {
self.total_quarantined.fetch_add(1, Ordering::Relaxed);
let mut records = self.quarantine_records();
records.seq += 1;
let seq = records.seq;
if !records.by_account.contains_key(&tx.user_id)
&& records.by_account.len() >= MAX_QUARANTINED_ACCOUNTS
{
let oldest = records
.by_account
.iter()
.min_by_key(|(_, q)| q.last_seq)
.map(|(k, _)| k.clone());
if let Some(oldest) = oldest {
records.by_account.remove(&oldest);
self.quarantine_records_evicted
.fetch_add(1, Ordering::Relaxed);
}
}
let entry = records.by_account.entry(tx.user_id.clone()).or_default();
entry.count += 1;
entry.last_seq = seq;
entry.rows.push_back(AccountQuarantinedRow {
request_id: visible_request_id(tx),
tx_type: tx.tx_type.clone(),
amount: tx.amount,
quarantined_at: quarantined_at.to_string(),
});
while entry.rows.len() > MAX_RECENT_QUARANTINED_PER_ACCOUNT {
entry.rows.pop_front();
}
}
#[cfg(test)]
pub(crate) fn note_quarantined_for_test(&self, tx: &BufferedTransaction) {
self.note_quarantined(tx, &chrono::Utc::now().to_rfc3339());
}
#[cfg(test)]
pub(crate) fn pending_snapshot(&self) -> Vec<BufferedTransaction> {
self.pending.iter().map(|e| e.value().clone()).collect()
}
fn drain(&self) -> (Vec<BufferedTransaction>, Vec<(String, f64)>) {
let mut txs = Vec::with_capacity(self.pending.len());
let keys: Vec<String> = self.pending.iter().map(|e| e.key().clone()).collect();
for key in keys {
if let Some((_, tx)) = self.pending.remove(&key) {
txs.push(tx);
}
}
let mut balances = Vec::with_capacity(self.balance_snapshots.len());
let bkeys: Vec<String> = self
.balance_snapshots
.iter()
.map(|e| e.key().clone())
.collect();
for key in bkeys {
if let Some((uid, bal)) = self.balance_snapshots.remove(&key) {
balances.push((uid, bal));
}
}
(txs, balances)
}
fn rejected_path(wal: &Wal) -> std::path::PathBuf {
let wal_path = wal.path();
let stem = wal_path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("wal");
let ext = wal_path
.extension()
.and_then(|s| s.to_str())
.unwrap_or("jsonl");
wal_path.with_file_name(format!("{stem}-rejected.{ext}"))
}
fn retired_path(wal: &Wal) -> std::path::PathBuf {
let wal_path = wal.path();
let stem = wal_path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("wal");
let ext = wal_path
.extension()
.and_then(|s| s.to_str())
.unwrap_or("jsonl");
wal_path.with_file_name(format!("{stem}-retired.{ext}"))
}
fn append_line(path: &std::path::Path, line: &serde_json::Value) {
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let write_result = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
.and_then(|mut f| writeln!(f, "{}", line));
if let Err(e) = write_result {
eprintln!(
" [FLUSH] Failed to write quarantine file {}: {}",
path.display(),
e
);
}
}
const MAX_QUARANTINE_BYTES: u64 = 64 * 1024 * 1024;
fn rotate_quarantine_if_full(path: &std::path::Path) {
let too_big = std::fs::metadata(path)
.map(|m| m.len() >= Self::MAX_QUARANTINE_BYTES)
.unwrap_or(false);
if !too_big {
return;
}
let rotated = path.with_extension(match path.extension().and_then(|e| e.to_str()) {
Some(ext) => format!("{ext}.1"),
None => "1".to_string(),
});
if std::fs::rename(path, &rotated).is_ok() {
eprintln!(
" [FLUSH] Quarantine file passed {} MB; rotated to {}",
Self::MAX_QUARANTINE_BYTES / (1024 * 1024),
rotated.display()
);
}
}
fn quarantine(&self, tx: &BufferedTransaction, wal: &Wal, error: &str, refused: bool) {
let quarantined_at = chrono::Utc::now().to_rfc3339();
self.note_quarantined(tx, &quarantined_at);
let job_name = job_name_for(tx);
let path = Self::rejected_path(wal);
let owed = !refused && wal.has_open_entry(&tx.request_id);
let dropped = !refused && !owed;
let line = serde_json::json!({
"request_id": tx.request_id,
"user_id": tx.user_id,
"job_name": job_name,
"tx_type": tx.tx_type,
"amount": tx.amount,
"worker_id": tx.worker_id,
"error": error,
"quarantined_at": quarantined_at,
"retired": refused,
"owed": owed,
"dropped": dropped,
});
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
Self::rotate_quarantine_if_full(&path);
Self::append_line(&path, &line);
eprintln!(
" [FLUSH] Quarantined {} (job \"{}\"){}: {}",
tx.request_id,
job_name,
if refused {
", retired"
} else if dropped {
", dropped (no WAL entry: nothing re-sends it)"
} else {
""
},
error
);
if refused || dropped {
Self::append_line(&Self::retired_path(wal), &line);
}
if refused {
self.total_retired.fetch_add(1, Ordering::Relaxed);
let _ = wal.update_status(&tx.request_id, WalStatus::Failed, None, None);
}
if dropped {
self.total_dropped.fetch_add(1, Ordering::Relaxed);
}
}
fn record_failure(&self, tx: BufferedTransaction, wal: &Wal, error: &str, refused: bool) {
let attempts = {
let mut entry = self.attempts.entry(tx.request_id.clone()).or_insert(0);
*entry += 1;
*entry
};
if attempts >= MAX_FLUSH_ATTEMPTS {
self.attempts.remove(&tx.request_id);
self.metadata.remove(&tx.request_id);
self.quarantine(&tx, wal, error, refused);
} else {
self.pending.insert(tx.request_id.clone(), tx);
}
}
fn record_success(&self, request_id: &str) {
self.attempts.remove(request_id);
self.metadata.remove(request_id);
}
pub fn flush_to_api(&self, api_url: &str, api_key: &str, wal: &Wal) -> usize {
self.last_flush_at.store(now_secs(), Ordering::Relaxed);
let (txs, _balances) = self.drain();
if txs.is_empty() {
return 0;
}
let mut src = txs.into_iter().map(|tx| {
let row = self.batch_row(&tx);
let size = serde_json::to_string(&row).map_or(0, |s| s.len());
((tx, row), size)
});
let mut carry = None;
let mut sent = 0;
let mut splits = MAX_SPLIT_REQUESTS_PER_FLUSH;
loop {
let chunk = fill_chunk(&mut src, &mut carry);
if chunk.is_empty() {
break;
}
let (txs, rows): (Vec<_>, Vec<_>) = chunk.into_iter().unzip();
sent += self.flush_chunk(api_url, api_key, wal, txs, rows, &mut splits);
}
sent
}
fn batch_row(&self, tx: &BufferedTransaction) -> serde_json::Value {
let dashboard_type = dashboard_type(tx.tx_type.as_str());
let job_name = job_name_for(tx);
let (amount, status) = match tx.tx_type.as_str() {
"cancel" => (0.0f64, "failed"),
"earning" | "refund" => (floor_to_hub_precision(tx.amount), "completed"),
_ => (tx.amount, "completed"),
};
let mut row = serde_json::json!({
"request_id": tx.request_id,
"zakuro_user_id": tx.user_id,
"job_name": job_name,
"transaction_type": dashboard_type,
"amount": amount,
"status": status,
"duration_ms": if tx.duration_ms > 0.0 { serde_json::Value::Number(serde_json::Number::from_f64(tx.duration_ms).unwrap()) } else { serde_json::Value::Null },
"worker_id": tx.worker_name,
"source_node": tx.source_node,
"compute_hours": if tx.duration_ms > 0.0 { Some(tx.duration_ms / 3_600_000.0) } else { None::<f64> },
"executor": tx.worker_name,
"destination": tx.worker_uri,
"price_per_hour": if tx.price_per_hour > 0.0 { Some(tx.price_per_hour) } else { None::<f64> },
"metadata": self.metadata.get(&tx.request_id).map(|m| m.value().clone())
});
super::ledger::attach_voucher_nonce(&mut row, tx.voucher_nonce.as_deref());
row
}
fn flush_chunk(
&self,
api_url: &str,
api_key: &str,
wal: &Wal,
txs: Vec<BufferedTransaction>,
payload: Vec<serde_json::Value>,
splits: &mut usize,
) -> usize {
let tx_count = txs.len();
let endpoint = format!("{}/api/broker/batch-sync", api_url.trim_end_matches('/'));
let payload_str = match serde_json::to_string(&payload) {
Ok(s) => s,
Err(e) => {
eprintln!(" [FLUSH] Failed to serialize payload: {}", e);
for tx in txs {
self.pending.insert(tx.request_id.clone(), tx);
}
return 0;
}
};
match ureq::post(&endpoint)
.config()
.http_status_as_error(false)
.build()
.header("X-Broker-Api-Key", api_key)
.header("Content-Type", "application/json")
.send(payload_str.as_str())
{
Ok(resp) if resp.status().as_u16() == 200 => {
let body = resp.into_body().read_to_string().unwrap_or_default();
let parsed: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
let inserted = parsed["inserted"].as_u64().unwrap_or(tx_count as u64);
let failed = parsed["failed"].as_u64().unwrap_or(0);
let success = parsed["success"].as_bool().unwrap_or(true);
self.total_flushed.fetch_add(inserted, Ordering::Relaxed);
match parsed["results"].as_array() {
Some(results) => {
let mut item_status: std::collections::HashMap<&str, (&str, &str)> =
std::collections::HashMap::new();
let mut item_receipt: std::collections::HashMap<&str, &str> =
std::collections::HashMap::new();
for r in results {
if let (Some(rid), Some(st)) =
(r["request_id"].as_str(), r["status"].as_str())
{
item_status.insert(rid, (st, r["error"].as_str().unwrap_or("")));
if let Some(h) =
r["receipt_hash"].as_str().filter(|h| !h.is_empty())
{
item_receipt.insert(rid, h);
}
}
}
let item_error_summary = summarize_item_errors(&parsed["errors"]);
for tx in &txs {
let (status, item_error) = item_status
.get(tx.request_id.as_str())
.copied()
.unwrap_or(("applied", ""));
match status {
"failed" => {
self.record_failure(
tx.clone(),
wal,
&item_error_summary,
is_permanent_item_error(item_error),
);
}
_ => {
self.record_success(&tx.request_id);
if let Some(h) = item_receipt.get(tx.request_id.as_str()) {
let _ = wal.attach_receipt(&tx.request_id, h);
}
retire_delivered(wal, tx);
}
}
}
}
None => {
for tx in &txs {
self.record_success(&tx.request_id);
retire_delivered(wal, tx);
}
}
}
if !success || failed > 0 {
eprintln!(
" [FLUSH] Partial sync to {}: {} inserted, {} failed",
api_url, inserted, failed
);
if let Some(errors) = parsed["errors"].as_array() {
for e in errors.iter().take(3) {
eprintln!(
" [FLUSH] → {}: {}",
e["job_name"].as_str().unwrap_or("?"),
e["error"].as_str().unwrap_or("unknown")
);
}
if errors.len() > 3 {
eprintln!(" [FLUSH] … and {} more errors", errors.len() - 3);
}
}
} else {
println!(
" [FLUSH] Synced {} transactions to {} via API",
inserted, api_url
);
}
inserted as usize
}
Ok(resp) => {
let status = resp.status().as_u16();
eprintln!(" [FLUSH] API sync failed: status {}", status);
if matches!(status, 413 | 422) && txs.len() > 1 {
let (mut left_tx, mut left_rows) = (txs, payload);
let mid = left_tx.len() / 2;
let right_tx = left_tx.split_off(mid);
let right_rows = left_rows.split_off(mid);
let mut sent = 0;
for (half_tx, half_rows) in [(left_tx, left_rows), (right_tx, right_rows)] {
if *splits == 0 {
for tx in half_tx {
self.pending.insert(tx.request_id.clone(), tx);
}
continue;
}
*splits -= 1;
sent += self.flush_chunk(api_url, api_key, wal, half_tx, half_rows, splits);
}
return sent;
}
let body = resp.into_body().read_to_string().unwrap_or_default();
if matches!(status, 413 | 422) {
eprintln!(
" [FLUSH] hub refused row {} as malformed (status {}): {}; \
kept owed, will be re-sent",
txs.first().map_or("", |t| t.request_id.as_str()),
status,
body
);
}
let parsed: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
let item_errors: std::collections::HashMap<&str, &str> = parsed["detail"]
["results"]
.as_array()
.map(|rs| {
rs.iter()
.filter_map(|r| {
Some((r["request_id"].as_str()?, r["error"].as_str().unwrap_or("")))
})
.collect()
})
.unwrap_or_default();
let error = format!("status {}", status);
for tx in txs {
let refused = item_errors
.get(tx.request_id.as_str())
.is_some_and(|item_error| is_permanent_item_error(item_error));
self.record_failure(tx, wal, &error, refused);
}
0
}
Err(e) => {
eprintln!(" [FLUSH] API sync failed: {}", e);
let error = e.to_string();
for tx in txs {
self.record_failure(tx, wal, &error, false);
}
0
}
}
}
}
impl std::fmt::Debug for TransactionBuffer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TransactionBuffer")
.field("pending", &self.pending.len())
.field("balance_snapshots", &self.balance_snapshots.len())
.field("total_flushed", &self.total_flushed())
.finish()
}
}
#[cfg(test)]
mod flush_batch_size_tests {
use super::*;
fn fat_tx(i: usize) -> BufferedTransaction {
BufferedTransaction {
request_id: format!("earn-{}-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", i),
user_id: "5492316032".into(),
tx_type: "earning".into(),
amount: 0.012112875187777778,
balance_after: 19998.863504120014,
worker_id: "a59edb75-a8e0-4822-a344-bbffcd791718".into(),
duration_ms: 2340.5812345,
source_node: Some("zc://node-6188100f5daf463e".into()),
worker_name: Some("worker-zc-worker-cloudcompute-larger-name-1".into()),
worker_uri: Some("http://10.13.13.20:3960/execute".into()),
price_per_hour: 3.6,
voucher_nonce: None,
}
}
#[test]
fn a_full_batch_stays_under_the_hub_body_limit() {
let txs: Vec<_> = (0..MAX_FLUSH_BATCH).map(fat_tx).collect();
let payload: Vec<serde_json::Value> = txs
.iter()
.map(|tx| {
serde_json::json!({
"request_id": tx.request_id,
"zakuro_user_id": tx.user_id,
"job_name": job_name_for(tx),
"transaction_type": dashboard_type(tx.tx_type.as_str()),
"amount": tx.amount,
"status": "completed",
"duration_ms": tx.duration_ms,
"worker_id": tx.worker_name,
"source_node": tx.source_node,
"compute_hours": tx.duration_ms / 3_600_000.0,
"executor": tx.worker_name,
"destination": tx.worker_uri,
"price_per_hour": tx.price_per_hour,
"metadata": null
})
})
.collect();
let bytes = serde_json::to_string(&payload).expect("serializes").len();
assert!(
bytes < HUB_MAX_BODY_BYTES / 2,
"a full batch is {bytes} bytes; the hub 413s over {HUB_MAX_BODY_BYTES} \
and the broker sees that as a broken pipe (#330)"
);
}
#[test]
fn chunks_of_metadata_rows_stay_under_the_byte_cap() {
for samples in [crate::sched::bill::MAX_TRUST_SAMPLES, 200] {
let buf = TransactionBuffer::new();
let payloads: Vec<Vec<u8>> = (0..samples as u32)
.map(|i| i.to_le_bytes().to_vec())
.collect();
for i in 0..MAX_FLUSH_BATCH {
let tx = fat_tx(i);
let rid = tx.request_id.clone();
buf.push_transaction(tx);
buf.attach_metadata(
&rid,
crate::sched::bill::trust_metadata(
payloads.iter().map(|p| (p.as_slice(), 123456.0)),
),
);
}
let mut src = buf.pending_snapshot().into_iter().map(|t| {
let row = buf.batch_row(&t);
let size = serde_json::to_string(&row).unwrap().len();
(row, size)
});
let mut carry = None;
let (mut n_chunks, mut covered) = (0, 0);
loop {
let chunk = fill_chunk(&mut src, &mut carry);
if chunk.is_empty() {
break;
}
n_chunks += 1;
covered += chunk.len();
let body = serde_json::to_string(&chunk).unwrap().len();
assert!(
body <= MAX_FLUSH_BYTES && chunk.len() <= MAX_FLUSH_BATCH,
"{samples} samples: a chunk body is {body} bytes / {} rows",
chunk.len()
);
}
assert!(n_chunks > 1, "{samples} samples: must split by bytes");
assert_eq!(covered, MAX_FLUSH_BATCH, "every row lands in a chunk");
}
}
fn chunk_all(sizes: &[usize]) -> Vec<Vec<usize>> {
let mut src = sizes.iter().copied().enumerate();
let mut carry = None;
let mut out = Vec::new();
loop {
let c = fill_chunk(&mut src, &mut carry);
if c.is_empty() {
return out;
}
out.push(c);
}
}
#[test]
fn chunking_edges() {
assert!(chunk_all(&[]).is_empty());
assert_eq!(chunk_all(&[10, 10]), vec![vec![0, 1]]);
let big = MAX_FLUSH_BYTES * 2;
assert_eq!(chunk_all(&[10, big, 10]), vec![vec![0], vec![1], vec![2]]);
assert_eq!(chunk_all(&vec![1; MAX_FLUSH_BATCH + 1]).len(), 2);
}
#[test]
fn at_most_one_chunk_of_rows_is_alive_at_a_time() {
let pulled = std::cell::Cell::new(0usize);
let mut src = (0..50_000usize).map(|i| {
pulled.set(pulled.get() + 1);
(i, 4_000) });
let mut carry = None;
let mut emitted = 0;
let mut chunks = 0;
loop {
let chunk = fill_chunk(&mut src, &mut carry);
if chunk.is_empty() {
break;
}
emitted += chunk.len();
chunks += 1;
assert!(chunk.len() < MAX_FLUSH_BATCH);
assert!(
pulled.get() <= emitted + 1,
"pulled {} rows but only {emitted} are in sent chunks",
pulled.get()
);
}
assert_eq!(emitted, 50_000);
assert!(chunks > 300);
}
#[test]
fn a_full_quarantine_file_is_rotated_not_grown_forever() {
let dir = std::env::temp_dir().join(format!("zc-quarantine-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("tmpdir");
let path = dir.join("wal-rejected.jsonl");
std::fs::write(&path, b"small\n").expect("write");
TransactionBuffer::rotate_quarantine_if_full(&path);
assert!(path.exists(), "a small file is not rotated");
assert!(
!dir.join("wal-rejected.jsonl.1").exists(),
"nothing rotated yet"
);
let big = vec![b'x'; TransactionBuffer::MAX_QUARANTINE_BYTES as usize + 1];
std::fs::write(&path, &big).expect("write big");
TransactionBuffer::rotate_quarantine_if_full(&path);
assert!(!path.exists(), "the full file was moved aside");
let rotated = dir.join("wal-rejected.jsonl.1");
assert!(rotated.exists(), "one previous generation is kept");
assert_eq!(
std::fs::metadata(&rotated).expect("rotated").len(),
big.len() as u64,
"the rotated file keeps its contents"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_backlog_is_split_into_bounded_requests() {
let backlog = 274_461; let chunks = (0..backlog)
.collect::<Vec<_>>()
.chunks(MAX_FLUSH_BATCH)
.len();
assert!(chunks > 1, "a backlog must not be one request");
assert!(
chunks * MAX_FLUSH_BATCH >= backlog,
"every transaction must land in some chunk"
);
}
}
#[cfg(test)]
mod earn_earning_payload_tests {
use super::*;
fn capturing_hub() -> (String, std::sync::mpsc::Receiver<String>) {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
let _ = tx.send(body);
let _ = req.respond(tiny_http::Response::from_string(
r#"{"success":true,"inserted":1,"failed":0}"#,
));
}
});
(format!("http://127.0.0.1:{port}"), rx)
}
#[test]
fn flush_sends_an_earning_transaction_as_earning() {
let path = format!(
"/tmp/zc_test_flush_dashboard_type_{}_{}.jsonl",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
);
let _ = std::fs::remove_file(&path);
let wal = Wal::open(&path).expect("open test WAL");
let buf = TransactionBuffer::new();
buf.push_transaction(BufferedTransaction {
request_id: "earn-r1".to_string(),
user_id: "owner".to_string(),
tx_type: "earning".to_string(),
amount: 1.5,
balance_after: 1.5,
worker_id: "w0".to_string(),
duration_ms: 10.0,
source_node: None,
worker_name: Some("w0".to_string()),
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
});
let (hub, bodies) = capturing_hub();
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 1);
let sent: serde_json::Value = serde_json::from_str(&bodies.recv().unwrap()).unwrap();
assert_eq!(
sent[0]["request_id"], "earn-r1",
"the earn row must be the one sent"
);
assert_eq!(
sent[0]["transaction_type"], "earning",
"an `earning` tx must be sent to the hub as `earning`, or the \
earnings summary silently drops it"
);
let _ = std::fs::remove_file(&path);
}
}
#[cfg(test)]
mod quarantine_tests {
use super::*;
fn test_wal(name: &str) -> (std::path::PathBuf, Wal) {
let path = std::path::PathBuf::from(format!(
"/tmp/zc_test_flush_quarantine_{}_{}_{}.jsonl",
name,
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_file(&path);
let wal = Wal::open(&path).expect("open test WAL");
(path, wal)
}
fn cleanup(wal_path: &std::path::Path, wal: &Wal) {
let _ = std::fs::remove_file(wal_path);
let _ = std::fs::remove_file(TransactionBuffer::rejected_path(wal));
let _ = std::fs::remove_file(TransactionBuffer::retired_path(wal));
}
fn tx(request_id: &str, user_id: &str) -> BufferedTransaction {
BufferedTransaction {
request_id: request_id.to_string(),
user_id: user_id.to_string(),
tx_type: "commit".to_string(),
amount: 1.0,
balance_after: 1.0,
worker_id: "w0".to_string(),
duration_ms: 10.0,
source_node: None,
worker_name: Some("w0".to_string()),
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
}
}
fn read_rejected_lines(wal: &Wal) -> Vec<serde_json::Value> {
let path = TransactionBuffer::rejected_path(wal);
match std::fs::read_to_string(&path) {
Ok(contents) => contents
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| serde_json::from_str(l).expect("quarantine line must be valid JSON"))
.collect(),
Err(_) => Vec::new(),
}
}
fn hub_always(status: u16) -> String {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
std::thread::spawn(move || {
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
let _ = req.respond(tiny_http::Response::from_string("").with_status_code(status));
}
});
format!("http://127.0.0.1:{port}")
}
fn dead_hub() -> String {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
drop(server); format!("http://127.0.0.1:{port}")
}
fn hub_fail_then_succeed(fail_times: usize, status: u16) -> String {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
std::thread::spawn(move || {
let mut seen = 0usize;
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
seen += 1;
if seen <= fail_times {
let _ =
req.respond(tiny_http::Response::from_string("").with_status_code(status));
} else {
let _ = req.respond(tiny_http::Response::from_string(
r#"{"success":true,"inserted":1,"failed":0}"#,
));
}
}
});
format!("http://127.0.0.1:{port}")
}
fn hub_with_item_failure(bad_rid: &'static str) -> String {
hub_with_item_failure_then_succeed(bad_rid, usize::MAX)
}
fn hub_with_item_failure_then_succeed(bad_rid: &'static str, fail_times: usize) -> String {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
std::thread::spawn(move || {
let mut seen = 0usize;
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
seen += 1;
let bad_still_fails = seen <= fail_times;
let parsed: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
let items = parsed.as_array().cloned().unwrap_or_default();
let results: Vec<serde_json::Value> = items
.iter()
.map(|item| {
let rid = item["request_id"].as_str().unwrap_or("");
let status = if rid == bad_rid && bad_still_fails {
"failed"
} else {
"applied"
};
serde_json::json!({"request_id": rid, "status": status})
})
.collect();
let failed = results.iter().filter(|r| r["status"] == "failed").count();
let inserted = results.len() - failed;
let resp_body = serde_json::json!({
"success": failed == 0,
"inserted": inserted,
"failed": failed,
"results": results,
"errors": if failed > 0 {
serde_json::json!([{"job_name": "Compute Job (w0)", "error": "User not found: anonymous"}])
} else {
serde_json::json!([])
},
})
.to_string();
let _ = req.respond(tiny_http::Response::from_string(resp_body));
}
});
format!("http://127.0.0.1:{port}")
}
fn executed(request_id: &str) -> crate::broker::wal::WalEntry {
crate::broker::wal::WalEntry {
request_id: request_id.to_string(),
user_id: "alice".to_string(),
reservation_id: format!("res-{request_id}"),
estimated_cost: 1.0,
actual_cost: Some(1.0),
worker_id: "w0".to_string(),
worker_name: Some("w0".to_string()),
duration_ms: Some(10.0),
timestamp: chrono::Utc::now(),
status: WalStatus::Executed,
voucher_nonce: None,
unit: crate::broker::wal::WAL_UNIT.to_string(),
payout: None,
hubless: false,
receipt_hash: None,
}
}
fn still_owed(wal: &Wal, request_id: &str) -> bool {
wal.read_uncommitted()
.unwrap()
.iter()
.any(|e| e.request_id == request_id)
}
#[test]
fn a_delivered_debit_retires_its_wal_entry() {
let (path, wal) = test_wal("debit_retired");
wal.append(&executed("r-debit")).unwrap();
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-debit", "alice"));
assert!(still_owed(&wal, "r-debit"), "owed until delivered");
let hub = hub_with_item_failure("nobody");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 1);
assert!(
!still_owed(&wal, "r-debit"),
"delivered, so no longer replayed"
);
cleanup(&path, &wal);
}
#[test]
fn a_delivered_debit_retires_its_wal_entry_on_an_older_hub() {
let (path, wal) = test_wal("debit_retired_legacy");
wal.append(&executed("r-old")).unwrap();
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-old", "alice"));
let hub = hub_fail_then_succeed(0, 200);
buf.flush_to_api(&hub, "key", &wal);
assert!(!still_owed(&wal, "r-old"));
cleanup(&path, &wal);
}
#[derive(Clone, Copy)]
enum FailShape {
Ok200,
AllFailed500,
}
fn hub_deciding_rows(
shape: FailShape,
refuse: impl Fn(&serde_json::Value) -> Option<String> + Send + 'static,
) -> String {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
std::thread::spawn(move || {
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
let parsed: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
let items = parsed.as_array().cloned().unwrap_or_default();
let mut errors = Vec::new();
let results: Vec<serde_json::Value> = items
.iter()
.map(|item| match refuse(item) {
Some(error) => {
errors.push(serde_json::json!({
"job_name": item["job_name"], "error": error,
}));
serde_json::json!({
"request_id": item["request_id"], "status": "failed",
"error": error, "receipt_hash": null,
})
}
None => serde_json::json!({
"request_id": item["request_id"], "status": "applied",
"error": null, "receipt_hash": null,
}),
})
.collect();
let failed = errors.len();
let body = serde_json::json!({
"results": results, "applied": results.len() - failed,
"duplicates": 0, "failed": failed,
"success": failed == 0, "inserted": results.len() - failed,
"errors": errors,
});
let (status, body) = match shape {
FailShape::AllFailed500 if failed == results.len() && failed > 0 => {
(500, serde_json::json!({ "detail": body }))
}
_ => (200, body),
};
let _ = req.respond(
tiny_http::Response::from_string(body.to_string()).with_status_code(status),
);
}
});
format!("http://127.0.0.1:{port}")
}
fn read_retired_lines(wal: &Wal) -> Vec<serde_json::Value> {
std::fs::read_to_string(TransactionBuffer::retired_path(wal))
.unwrap_or_default()
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| serde_json::from_str(l).unwrap())
.collect()
}
fn owed_after_quarantine_for(error: &str) -> bool {
let (path, wal) = test_wal("reason");
wal.append(&executed("r-refused")).unwrap();
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-refused", "alice"));
let refusal = error.to_string();
let hub = hub_deciding_rows(FailShape::Ok200, move |_| Some(refusal.clone()));
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key", &wal);
}
assert_eq!(buf.pending_count(), 0, "quarantined: {error}");
let rejected = read_rejected_lines(&wal);
assert_eq!(rejected.len(), 1, "{error}");
let owed = still_owed(&wal, "r-refused");
assert_eq!(rejected[0]["retired"], !owed, "{error}");
assert_eq!(
read_retired_lines(&wal).len(),
usize::from(!owed),
"{error}"
);
assert_eq!(buf.total_retired(), u64::from(!owed), "{error}");
cleanup(&path, &wal);
owed
}
macro_rules! permanent_refusal_retires {
($($name:ident: $error:expr;)*) => {$(
#[test]
fn $name() {
assert!(!owed_after_quarantine_for($error), "{}", $error);
}
)*};
}
permanent_refusal_retires! {
permanent_refusal_unrecorded_job_shape_retires:
"403: a user key may post credit_purchase only for a recorded job \
(request_id earn-<rid> or <rid>-earn-refund) (#403)";
permanent_refusal_charge_names_no_worker_retires:
"403: the charge for job-7 names no worker, so no provider can claim it (#403)";
permanent_refusal_refund_to_another_account_retires:
"403: a refund may go only to the account that was charged (#403)";
permanent_refusal_payouts_exceed_charge_retires:
"403: earnings and refunds for job-7 would exceed its charge 1.000000 (#403)";
permanent_refusal_service_only_type_retires:
"403: transaction_type 'credit' requires a service credential";
}
#[test]
fn refusals_that_can_clear_keep_the_entry_owed() {
for error in [
"403: an earning goes only to the owner of the worker that ran the job (#403)",
"403: a user key may post earnings only to its own account (#403)",
"403: Acting for another account requires a hub voucher for that account (no_proof)",
"403: Acting for another account requires a hub voucher for that account (unknown_voucher)",
"403: earning names worker w-1, the job ran on w-2 (#403)",
"403: Acting for another account requires a hub voucher for that account (lookup_error)",
"402: Insufficient balance",
"403: would exceed its charge earnings and refunds for job-7 (#403)",
] {
assert!(owed_after_quarantine_for(error), "{error}");
}
}
#[test]
fn a_wrong_key_earning_under_binding_enforcement_stays_owed() {
let (path, wal) = test_wal("wrong_key_earn");
let mut earned = executed("earn-job-1");
earned.user_id = "owner".to_string();
earned.status = WalStatus::Earned;
wal.append(&earned).unwrap();
let buf = TransactionBuffer::new();
let mut earn = tx("earn-job-1", "owner");
earn.tx_type = "earning".to_string();
buf.push_transaction(earn);
let key_owner = "intruder";
let enforce = true;
let hub = hub_deciding_rows(FailShape::AllFailed500, move |row| {
let target = row["zakuro_user_id"].as_str().unwrap_or("");
if enforce && key_owner != target && row.get("voucher_nonce").is_none() {
return Some(
"403: Acting for another account requires a hub voucher \
for that account (no_proof)"
.to_string(),
);
}
if row["transaction_type"] == "earning" && key_owner != target {
return Some(
"403: a user key may post earnings only to its own account (#403)".to_string(),
);
}
None
});
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key-of-intruder", &wal);
}
assert_eq!(buf.pending_count(), 0, "quarantined");
assert!(
still_owed(&wal, "earn-job-1"),
"the owner's earning is still owed"
);
assert_eq!(read_rejected_lines(&wal)[0]["retired"], false);
assert!(read_retired_lines(&wal).is_empty());
assert_eq!(buf.total_retired(), 0);
cleanup(&path, &wal);
}
#[test]
fn an_all_failed_500_chunk_is_classified_per_row() {
let (path, wal) = test_wal("all_failed_500");
wal.append(&executed("r-perm")).unwrap();
wal.append(&executed("r-trans")).unwrap();
let mut earned = executed("earn-r-x");
earned.status = WalStatus::Earned;
wal.append(&earned).unwrap();
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-perm", "alice"));
buf.push_transaction(tx("r-trans", "alice"));
let mut earn = tx("earn-r-x", "alice");
earn.tx_type = "earning".to_string();
buf.push_transaction(earn);
let hub = hub_deciding_rows(FailShape::AllFailed500, |row| {
Some(match row["request_id"].as_str().unwrap_or("") {
"r-trans" => "402: Insufficient balance".to_string(),
"earn-r-x" => {
"403: earnings and refunds for r-x would exceed its charge 1.000000 (#403)"
.to_string()
}
_ => "User not found: alice".to_string(),
})
});
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key", &wal);
}
assert_eq!(buf.pending_count(), 0);
assert!(!still_owed(&wal, "r-perm"));
assert!(!still_owed(&wal, "earn-r-x"));
assert!(still_owed(&wal, "r-trans"));
let retired: Vec<String> = read_retired_lines(&wal)
.iter()
.map(|l| l["request_id"].as_str().unwrap().to_string())
.collect();
assert_eq!(retired.len(), 2, "{retired:?}");
assert!(
retired.contains(&"r-perm".to_string()) && retired.contains(&"earn-r-x".to_string())
);
assert_eq!(buf.total_retired(), 2);
for line in read_rejected_lines(&wal) {
assert_eq!(line["retired"], line["request_id"] != "r-trans", "{line}");
}
cleanup(&path, &wal);
}
#[test]
fn a_quarantined_row_without_a_wal_entry_is_marked_dropped() {
let (path, wal) = test_wal("dropped_refund");
wal.append(&executed("r-debit")).unwrap();
let buf = TransactionBuffer::new();
let mut refund = tx("r-debit-earn-refund", "alice");
refund.tx_type = "refund".to_string();
buf.push_transaction(refund);
buf.push_transaction(tx("r-debit", "alice"));
let hub = hub_deciding_rows(FailShape::AllFailed500, |_| {
Some("500: database unavailable".to_string())
});
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key", &wal);
}
assert_eq!(buf.pending_count(), 0);
let rejected = read_rejected_lines(&wal);
let line = |rid: &str| {
rejected
.iter()
.find(|l| l["request_id"] == rid)
.cloned()
.unwrap_or_else(|| panic!("{rid} not quarantined: {rejected:?}"))
};
let refund_line = line("r-debit-earn-refund");
assert_eq!(refund_line["retired"], false);
assert_eq!(refund_line["owed"], false);
assert_eq!(refund_line["dropped"], true);
let debit_line = line("r-debit");
assert_eq!(debit_line["owed"], true);
assert_eq!(debit_line["dropped"], false);
assert!(still_owed(&wal, "r-debit"));
let retired = read_retired_lines(&wal);
assert_eq!(retired.len(), 1, "{retired:?}");
assert_eq!(retired[0]["request_id"], "r-debit-earn-refund");
assert_eq!(retired[0]["dropped"], true);
assert_eq!(buf.total_dropped(), 1);
assert_eq!(buf.total_retired(), 0);
cleanup(&path, &wal);
}
#[test]
fn the_retired_record_is_not_rotated_with_the_quarantine_file() {
let (path, wal) = test_wal("retired_rotation");
wal.append(&executed("r-gone")).unwrap();
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-gone", "alice"));
let hub = hub_deciding_rows(FailShape::Ok200, |_| Some("User not found: alice".into()));
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key", &wal);
}
let rejected = TransactionBuffer::rejected_path(&wal);
std::fs::OpenOptions::new()
.write(true)
.open(&rejected)
.unwrap()
.set_len(TransactionBuffer::MAX_QUARANTINE_BYTES)
.unwrap();
TransactionBuffer::rotate_quarantine_if_full(&rejected);
assert!(!rejected.exists(), "rotated away");
assert_eq!(read_retired_lines(&wal).len(), 1);
assert_eq!(read_retired_lines(&wal)[0]["request_id"], "r-gone");
cleanup(&path, &wal);
let _ = std::fs::remove_file(rejected.with_extension("jsonl.1"));
}
#[test]
fn payouts_are_floored_to_the_hubs_precision() {
assert_eq!(floor_to_hub_precision(1.0000004), 1.0);
assert_eq!(floor_to_hub_precision(0.9), 0.9);
assert_eq!(floor_to_hub_precision(0.9000009414), 0.900000);
assert_eq!(floor_to_hub_precision(2.5), 2.5);
assert_eq!(floor_to_hub_precision(0.0), 0.0);
let buf = TransactionBuffer::new();
for (kind, sent) in [("earning", 1.0), ("refund", 1.0), ("commit", 1.0000004)] {
let mut t = tx("r", "alice");
t.tx_type = kind.to_string();
t.amount = 1.0000004;
assert_eq!(buf.batch_row(&t)["amount"], sent, "{kind}");
}
}
#[test]
fn a_rejected_debit_stays_owed() {
let (path, wal) = test_wal("debit_rejected");
wal.append(&executed("r-bad")).unwrap();
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-bad", "alice"));
let hub = hub_with_item_failure("r-bad");
buf.flush_to_api(&hub, "key", &wal);
assert!(still_owed(&wal, "r-bad"));
cleanup(&path, &wal);
}
#[test]
fn whole_request_failure_quarantines_after_max_attempts() {
let (path, wal) = test_wal("always_500");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-bad", "alice"));
let hub = hub_always(500);
for _ in 0..MAX_FLUSH_ATTEMPTS {
assert_eq!(buf.pending_count(), 1, "not yet quarantined");
let flushed = buf.flush_to_api(&hub, "key", &wal);
assert_eq!(flushed, 0);
}
assert_eq!(buf.pending_count(), 0, "must be gone from pending");
let rejected = read_rejected_lines(&wal);
assert_eq!(
rejected.len(),
1,
"the quarantine event (and its one warning) must fire exactly once, \
not once per attempt"
);
assert_eq!(rejected[0]["request_id"], "r-bad");
cleanup(&path, &wal);
}
#[test]
fn settlement_backlog_is_attributed_per_account() {
let (path, wal) = test_wal("per_account");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-alice", "alice"));
buf.push_transaction(tx("r-bob", "bob"));
let hub = hub_always(500);
buf.flush_to_api(&hub, "key", &wal);
let alice = buf.account_settlement("alice");
assert_eq!(alice.settlement_pending, 1);
assert_eq!(alice.pending[0].request_id.as_deref(), Some("r-alice"));
assert_eq!(alice.pending[0].attempts, 1, "one failed flush so far");
assert_eq!(alice.settlement_quarantined, 0);
for _ in 1..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key", &wal);
}
assert_eq!(buf.total_quarantined(), 2);
for (account, own) in [("alice", "r-alice"), ("bob", "r-bob")] {
let view = buf.account_settlement(account);
assert_eq!(view.settlement_pending, 0);
assert_eq!(view.settlement_quarantined, 1, "{account}");
let ids: Vec<&str> = view
.quarantined
.iter()
.filter_map(|r| r.request_id.as_deref())
.collect();
assert_eq!(ids, vec![own], "{account} sees exactly its own row");
}
let carol = buf.account_settlement("carol");
assert_eq!(carol.settlement_quarantined, 0);
assert!(carol.quarantined.is_empty() && carol.pending.is_empty());
cleanup(&path, &wal);
}
#[test]
fn settlement_quarantined_rows_are_bounded_but_counted_exactly() {
let buf = TransactionBuffer::new();
let n = MAX_RECENT_QUARANTINED_PER_ACCOUNT + 7;
for i in 0..n {
buf.note_quarantined_for_test(&tx(&format!("r-{i:03}"), "alice"));
}
let view = buf.account_settlement("alice");
assert_eq!(view.settlement_quarantined, n as u64);
assert_eq!(view.quarantined.len(), MAX_RECENT_QUARANTINED_PER_ACCOUNT);
assert_eq!(
view.quarantined.last().unwrap().request_id,
Some(format!("r-{:03}", n - 1)),
"newest rows are the ones kept"
);
}
#[test]
fn settlement_quarantine_accounts_are_capped_oldest_evicted() {
let buf = TransactionBuffer::new();
for i in 0..MAX_QUARANTINED_ACCOUNTS {
buf.note_quarantined_for_test(&tx(&format!("r-{i}"), &format!("u{i}")));
}
buf.note_quarantined_for_test(&tx("r-0b", "u0"));
assert_eq!(
buf.quarantine_records_evicted(),
0,
"at the cap, not past it"
);
assert!(!buf.account_settlement("u1").quarantine_incomplete);
buf.note_quarantined_for_test(&tx("r-new", "newcomer"));
assert_eq!(
buf.quarantine_records().by_account.len(),
MAX_QUARANTINED_ACCOUNTS
);
assert_eq!(
buf.account_settlement("u1").settlement_quarantined,
0,
"evicted"
);
assert_eq!(
buf.account_settlement("u0").settlement_quarantined,
2,
"kept"
);
assert_eq!(buf.account_settlement("newcomer").settlement_quarantined, 1);
assert_eq!(
buf.total_quarantined(),
MAX_QUARANTINED_ACCOUNTS as u64 + 2,
"the broker-wide /health count is not affected by eviction"
);
assert_eq!(buf.quarantine_records_evicted(), 1);
for account in ["u1", "u0", "newcomer", "never-quarantined"] {
assert!(
buf.account_settlement(account).quarantine_incomplete,
"{account}: every view is flagged once any record was evicted"
);
}
}
#[test]
fn concurrent_quarantines_never_overshoot_the_account_cap() {
let buf = std::sync::Arc::new(TransactionBuffer::new());
let threads = 16;
let per_thread = MAX_QUARANTINED_ACCOUNTS / 4;
let handles: Vec<_> = (0..threads)
.map(|t| {
let buf = buf.clone();
std::thread::spawn(move || {
for i in 0..per_thread {
buf.note_quarantined_for_test(&tx(
&format!("r-{t}-{i}"),
&format!("u{t}-{i}"),
));
assert!(
buf.quarantine_records().by_account.len() <= MAX_QUARANTINED_ACCOUNTS,
"cap overshot"
);
}
})
})
.collect();
for h in handles {
h.join().unwrap();
}
let accounts = threads * per_thread;
assert_eq!(
buf.quarantine_records().by_account.len(),
MAX_QUARANTINED_ACCOUNTS
);
assert_eq!(
buf.quarantine_records_evicted(),
(accounts - MAX_QUARANTINED_ACCOUNTS) as u64
);
assert_eq!(buf.total_quarantined(), accounts as u64);
}
#[test]
fn settlement_view_is_complete_until_a_record_is_evicted() {
let buf = TransactionBuffer::new();
buf.note_quarantined_for_test(&tx("r-1", "alice"));
let view = buf.account_settlement("bob");
assert_eq!(view.settlement_quarantined, 0);
assert!(!view.quarantine_incomplete);
assert_eq!(buf.quarantine_records_evicted(), 0);
}
#[test]
fn settlement_earning_rows_hide_the_requesters_request_id() {
let buf = TransactionBuffer::new();
let mut earn = tx("earn-req-of-alice", "owner");
earn.tx_type = "earning".to_string();
buf.push_transaction(earn.clone());
buf.note_quarantined_for_test(&earn);
let view = buf.account_settlement("owner");
assert_eq!(view.settlement_pending, 1);
assert_eq!(view.pending[0].request_id, None);
assert_eq!(view.quarantined[0].request_id, None);
let raw = serde_json::to_string(&view).unwrap();
assert!(!raw.contains("req-of-alice"), "{raw}");
assert!(!raw.contains("\"request_id\""), "{raw}");
}
#[test]
fn quarantined_entry_does_not_block_later_transactions() {
let (path, wal) = test_wal("unblocks_queue");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-bad", "alice"));
let bad_hub = hub_always(500);
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&bad_hub, "key", &wal);
}
assert_eq!(buf.pending_count(), 0, "bad tx quarantined out of pending");
buf.push_transaction(tx("r-good", "bob"));
let good_hub_url = {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
std::thread::spawn(move || {
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
let _ = req.respond(tiny_http::Response::from_string(
r#"{"success":true,"inserted":1,"failed":0}"#,
));
}
});
format!("http://127.0.0.1:{port}")
};
let flushed = buf.flush_to_api(&good_hub_url, "key", &wal);
assert_eq!(flushed, 1, "the later transaction must reach the hub");
assert_eq!(buf.pending_count(), 0);
cleanup(&path, &wal);
}
#[test]
fn transport_error_counts_as_an_attempt_and_quarantines() {
let (path, wal) = test_wal("transport_error");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-unreachable", "alice"));
let hub = dead_hub();
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key", &wal);
}
assert_eq!(buf.pending_count(), 0);
let rejected = read_rejected_lines(&wal);
assert_eq!(rejected.len(), 1);
assert_eq!(rejected[0]["request_id"], "r-unreachable");
cleanup(&path, &wal);
}
#[test]
fn per_item_rejection_inside_200_gets_the_same_attempt_grace() {
let (path, wal) = test_wal("per_item");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-bad-item", "anonymous-ish"));
buf.push_transaction(tx("r-good-item", "bob"));
let hub = hub_with_item_failure("r-bad-item");
for i in 0..MAX_FLUSH_ATTEMPTS {
let flushed = buf.flush_to_api(&hub, "key", &wal);
if i == 0 {
assert_eq!(
flushed, 1,
"the good item in the same batch must still flush on the first attempt"
);
}
}
assert_eq!(buf.pending_count(), 0, "must be gone from pending");
let rejected = read_rejected_lines(&wal);
assert_eq!(
rejected.len(),
1,
"quarantined exactly once, after MAX_FLUSH_ATTEMPTS, not on the first failure"
);
assert_eq!(rejected[0]["request_id"], "r-bad-item");
assert!(
rejected[0]["error"]
.as_str()
.unwrap_or("")
.contains("User not found"),
"the hub's per-item error text must be preserved"
);
cleanup(&path, &wal);
}
#[test]
fn per_item_rejection_that_later_succeeds_is_not_quarantined() {
let (path, wal) = test_wal("per_item_transient");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-transient-item", "bob"));
let hub = hub_with_item_failure_then_succeed("r-transient-item", 2);
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 0); assert_eq!(buf.pending_count(), 1, "requeued, not quarantined");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 0); assert_eq!(buf.pending_count(), 1, "requeued, not quarantined");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 1);
assert_eq!(buf.pending_count(), 0);
assert!(
read_rejected_lines(&wal).is_empty(),
"a per-item failure that eventually succeeds must never be quarantined"
);
cleanup(&path, &wal);
}
#[test]
fn transient_failure_then_success_is_not_quarantined() {
let (path, wal) = test_wal("transient");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-transient", "alice"));
let hub = hub_fail_then_succeed(2, 500);
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 0); assert_eq!(buf.pending_count(), 1, "requeued, not quarantined");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 0); assert_eq!(buf.pending_count(), 1, "requeued, not quarantined");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 1);
assert_eq!(buf.pending_count(), 0);
assert!(
read_rejected_lines(&wal).is_empty(),
"a transaction that eventually succeeds must never be quarantined"
);
cleanup(&path, &wal);
}
#[test]
fn anonymous_sentinel_is_never_buffered() {
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-anon", ANONYMOUS_USER));
assert_eq!(
buf.pending_count(),
0,
"the anonymous sentinel must never be buffered for the hub"
);
buf.push_transaction(tx("r-real", "alice"));
assert_eq!(
buf.pending_count(),
1,
"a normal user must still be buffered"
);
}
#[test]
fn quarantine_file_lines_are_valid_json_with_error_text() {
let (path, wal) = test_wal("valid_json_lines");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-one", "alice"));
buf.push_transaction(tx("r-two", "alice"));
let hub = hub_always(500);
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key", &wal);
}
let rejected_path = TransactionBuffer::rejected_path(&wal);
let contents = std::fs::read_to_string(&rejected_path).expect("quarantine file exists");
let lines: Vec<&str> = contents.lines().filter(|l| !l.trim().is_empty()).collect();
assert_eq!(lines.len(), 2, "one line per quarantined transaction");
for line in &lines {
let parsed: serde_json::Value =
serde_json::from_str(line).expect("each line must be standalone valid JSON");
assert!(
parsed["error"].as_str().is_some_and(|s| !s.is_empty()),
"each quarantined line must carry the error text"
);
}
cleanup(&path, &wal);
}
}
#[cfg(test)]
mod voucher_nonce_tests {
use super::*;
fn tx(nonce: Option<&str>) -> BufferedTransaction {
BufferedTransaction {
request_id: "r".into(),
user_id: "u".into(),
tx_type: "commit".into(),
amount: 1.0,
balance_after: 9.0,
worker_id: "w".into(),
duration_ms: 10.0,
source_node: None,
worker_name: Some("w".into()),
worker_uri: None,
price_per_hour: 1.0,
voucher_nonce: nonce.map(String::from),
}
}
#[test]
fn batch_row_carries_the_nonce_only_when_present() {
let buf = TransactionBuffer::new();
let row = buf.batch_row(&tx(Some("n-1")));
assert_eq!(row["voucher_nonce"], "n-1");
assert!(buf.batch_row(&tx(None)).get("voucher_nonce").is_none());
assert!(buf.batch_row(&tx(Some(""))).get("voucher_nonce").is_none());
}
#[test]
fn buffered_transaction_round_trips_and_old_json_decodes() {
let back: BufferedTransaction =
serde_json::from_str(&serde_json::to_string(&tx(Some("n-2"))).unwrap()).unwrap();
assert_eq!(back.voucher_nonce.as_deref(), Some("n-2"));
let plain = serde_json::to_string(&tx(None)).unwrap();
assert!(!plain.contains("voucher_nonce"));
let old = r#"{"request_id":"r","user_id":"u","tx_type":"commit","amount":1.0,"balance_after":2.0,"worker_id":"w","duration_ms":1.0,"source_node":null,"worker_name":null,"worker_uri":null,"price_per_hour":0.0}"#;
let back: BufferedTransaction = serde_json::from_str(old).unwrap();
assert_eq!(back.voucher_nonce, None);
}
}