use dashmap::DashMap;
use serde::{Deserialize, Serialize};
use std::io::Write;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use super::node_identity::now_secs;
use super::wal::{Wal, WalStatus};
pub const ANONYMOUS_USER: &str = "anonymous";
const MAX_FLUSH_ATTEMPTS: u32 = 5;
const MAX_FLUSH_BATCH: usize = 500;
const MAX_FLUSH_BYTES: usize = 512 * 1024;
fn fill_chunk<T>(
src: &mut impl Iterator<Item = (T, usize)>,
carry: &mut Option<(T, usize)>,
) -> Vec<T> {
let mut out = Vec::new();
let mut bytes = 2usize; while let Some((item, size)) = carry.take().or_else(|| src.next()) {
let add = size + 1; if !out.is_empty() && (out.len() >= MAX_FLUSH_BATCH || bytes + add > MAX_FLUSH_BYTES) {
*carry = Some((item, size));
break;
}
bytes += add;
out.push(item);
}
out
}
fn retire_delivered(wal: &Wal, tx: &BufferedTransaction) {
let _ = wal.update_status(
&tx.request_id,
WalStatus::Committed,
Some(tx.amount),
Some(tx.duration_ms),
);
}
#[cfg(test)]
mod settlement_liveness_tests {
use super::*;
#[test]
fn a_fresh_buffer_is_not_already_stale() {
let buf = TransactionBuffer::new();
assert!(buf.seconds_since_flush() < 5);
}
#[test]
fn an_idle_broker_still_proves_the_flush_path_runs() {
let buf = TransactionBuffer::new();
let path = std::env::temp_dir().join(format!(
"zc-liveness-{}.wal",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_file(&path);
let wal = Wal::open(&path).expect("open test WAL");
buf.last_flush_at.store(0, Ordering::Relaxed);
assert!(buf.seconds_since_flush() > 1000, "fixture must look stale");
assert_eq!(buf.pending_count(), 0);
buf.flush_to_api("http://127.0.0.1:1", "k", &wal);
assert!(
buf.seconds_since_flush() < 5,
"an empty flush must still stamp liveness"
);
let _ = std::fs::remove_file(&path);
}
#[test]
fn pending_work_is_visible_before_it_is_ever_quarantined() {
let buf = TransactionBuffer::new();
buf.push_transaction(BufferedTransaction {
request_id: "r1".into(),
user_id: "u1".into(),
tx_type: "commit".into(),
amount: 0.003,
balance_after: 1.0,
worker_id: "w1".into(),
duration_ms: 1.0,
source_node: None,
worker_name: None,
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
});
assert_eq!(buf.pending_count(), 1);
assert_eq!(buf.total_quarantined(), 0, "nothing failed — nothing ran");
}
#[test]
fn metadata_for_a_transaction_that_was_never_queued_is_not_kept() {
let buf = TransactionBuffer::new();
buf.attach_metadata("never-queued", serde_json::json!({"tasks": []}));
assert!(buf.metadata.is_empty());
buf.push_transaction(BufferedTransaction {
request_id: "r1".into(),
user_id: "u1".into(),
tx_type: "commit".into(),
amount: 0.003,
balance_after: 1.0,
worker_id: "w1".into(),
duration_ms: 1.0,
source_node: None,
worker_name: None,
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
});
buf.attach_metadata("r1", serde_json::json!({"tasks": []}));
assert_eq!(buf.metadata.len(), 1);
}
}
#[cfg(test)]
const HUB_MAX_BODY_BYTES: usize = 1024 * 1024;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BufferedTransaction {
pub request_id: String,
pub user_id: String,
pub tx_type: String,
pub amount: f64,
pub balance_after: f64,
pub worker_id: String,
pub duration_ms: f64,
pub source_node: Option<String>,
pub worker_name: Option<String>,
pub worker_uri: Option<String>,
pub price_per_hour: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub voucher_nonce: Option<String>,
}
pub struct TransactionBuffer {
pending: DashMap<String, BufferedTransaction>,
balance_snapshots: DashMap<String, f64>,
total_flushed: AtomicU64,
total_quarantined: AtomicU64,
last_flush_at: AtomicU64,
attempts: DashMap<String, u32>,
metadata: DashMap<String, serde_json::Value>,
anonymous_warned: AtomicBool,
}
pub(crate) fn dashboard_type(tx_type: &str) -> &str {
match tx_type {
"commit" | "cancel" => "job_execution",
"credit" => "credit_purchase",
other => other,
}
}
fn job_name_for(tx: &BufferedTransaction) -> String {
match tx.tx_type.as_str() {
"cancel" => format!(
"Cancelled Job ({})",
tx.worker_name.as_deref().unwrap_or(&tx.worker_id)
),
_ => format!(
"Compute Job ({})",
tx.worker_name.as_deref().unwrap_or(&tx.worker_id)
),
}
}
fn summarize_item_errors(errors: &serde_json::Value) -> String {
match errors.as_array() {
Some(errors) if !errors.is_empty() => {
let joined: Vec<String> = errors
.iter()
.take(3)
.map(|e| {
format!(
"{}: {}",
e["job_name"].as_str().unwrap_or("?"),
e["error"].as_str().unwrap_or("unknown")
)
})
.collect();
let mut summary = format!(
"hub reported {} item failure(s): {}",
errors.len(),
joined.join("; ")
);
if errors.len() > 3 {
summary.push_str(&format!(" … and {} more", errors.len() - 3));
}
summary
}
_ => "hub reported this item as failed (no error detail provided)".to_string(),
}
}
impl TransactionBuffer {
pub fn new() -> Self {
Self {
pending: DashMap::new(),
balance_snapshots: DashMap::new(),
total_flushed: AtomicU64::new(0),
total_quarantined: AtomicU64::new(0),
last_flush_at: AtomicU64::new(now_secs()),
attempts: DashMap::new(),
metadata: DashMap::new(),
anonymous_warned: AtomicBool::new(false),
}
}
pub fn push_transaction(&self, tx: BufferedTransaction) {
if tx.user_id == ANONYMOUS_USER {
if !self.anonymous_warned.swap(true, Ordering::Relaxed) {
eprintln!(
" [FLUSH] Skipping hub sync for anonymous-user transaction {} \
(local execution; nothing to settle)",
tx.request_id
);
}
return;
}
self.pending.insert(tx.request_id.clone(), tx);
}
pub fn attach_metadata(&self, request_id: &str, metadata: serde_json::Value) {
if self.pending.contains_key(request_id) {
self.metadata.insert(request_id.to_string(), metadata);
}
}
pub fn snapshot_balance(&self, user_id: &str, balance: f64) {
self.balance_snapshots.insert(user_id.to_string(), balance);
}
pub fn pending_count(&self) -> usize {
self.pending.len()
}
pub fn total_flushed(&self) -> u64 {
self.total_flushed.load(Ordering::Relaxed)
}
pub fn seconds_since_flush(&self) -> u64 {
now_secs().saturating_sub(self.last_flush_at.load(Ordering::Relaxed))
}
pub fn total_quarantined(&self) -> u64 {
self.total_quarantined.load(Ordering::Relaxed)
}
#[cfg(test)]
pub(crate) fn pending_snapshot(&self) -> Vec<BufferedTransaction> {
self.pending.iter().map(|e| e.value().clone()).collect()
}
fn drain(&self) -> (Vec<BufferedTransaction>, Vec<(String, f64)>) {
let mut txs = Vec::with_capacity(self.pending.len());
let keys: Vec<String> = self.pending.iter().map(|e| e.key().clone()).collect();
for key in keys {
if let Some((_, tx)) = self.pending.remove(&key) {
txs.push(tx);
}
}
let mut balances = Vec::with_capacity(self.balance_snapshots.len());
let bkeys: Vec<String> = self
.balance_snapshots
.iter()
.map(|e| e.key().clone())
.collect();
for key in bkeys {
if let Some((uid, bal)) = self.balance_snapshots.remove(&key) {
balances.push((uid, bal));
}
}
(txs, balances)
}
fn rejected_path(wal: &Wal) -> std::path::PathBuf {
let wal_path = wal.path();
let stem = wal_path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("wal");
let ext = wal_path
.extension()
.and_then(|s| s.to_str())
.unwrap_or("jsonl");
wal_path.with_file_name(format!("{stem}-rejected.{ext}"))
}
const MAX_QUARANTINE_BYTES: u64 = 64 * 1024 * 1024;
fn rotate_quarantine_if_full(path: &std::path::Path) {
let too_big = std::fs::metadata(path)
.map(|m| m.len() >= Self::MAX_QUARANTINE_BYTES)
.unwrap_or(false);
if !too_big {
return;
}
let rotated = path.with_extension(match path.extension().and_then(|e| e.to_str()) {
Some(ext) => format!("{ext}.1"),
None => "1".to_string(),
});
if std::fs::rename(path, &rotated).is_ok() {
eprintln!(
" [FLUSH] Quarantine file passed {} MB; rotated to {}",
Self::MAX_QUARANTINE_BYTES / (1024 * 1024),
rotated.display()
);
}
}
fn quarantine(&self, tx: &BufferedTransaction, wal: &Wal, error: &str) {
self.total_quarantined.fetch_add(1, Ordering::Relaxed);
let job_name = job_name_for(tx);
let path = Self::rejected_path(wal);
let line = serde_json::json!({
"request_id": tx.request_id,
"user_id": tx.user_id,
"job_name": job_name,
"tx_type": tx.tx_type,
"amount": tx.amount,
"worker_id": tx.worker_id,
"error": error,
"quarantined_at": chrono::Utc::now().to_rfc3339(),
});
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
Self::rotate_quarantine_if_full(&path);
let write_result = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&path)
.and_then(|mut f| writeln!(f, "{}", line));
if let Err(e) = write_result {
eprintln!(
" [FLUSH] Failed to write quarantine file {}: {}",
path.display(),
e
);
}
eprintln!(
" [FLUSH] Quarantined {} (job \"{}\"): {}",
tx.request_id, job_name, error
);
}
fn record_failure(&self, tx: BufferedTransaction, wal: &Wal, error: &str) {
let attempts = {
let mut entry = self.attempts.entry(tx.request_id.clone()).or_insert(0);
*entry += 1;
*entry
};
if attempts >= MAX_FLUSH_ATTEMPTS {
self.attempts.remove(&tx.request_id);
self.metadata.remove(&tx.request_id);
self.quarantine(&tx, wal, error);
} else {
self.pending.insert(tx.request_id.clone(), tx);
}
}
fn record_success(&self, request_id: &str) {
self.attempts.remove(request_id);
self.metadata.remove(request_id);
}
pub fn flush_to_api(&self, api_url: &str, api_key: &str, wal: &Wal) -> usize {
self.last_flush_at.store(now_secs(), Ordering::Relaxed);
let (txs, _balances) = self.drain();
if txs.is_empty() {
return 0;
}
let mut src = txs.into_iter().map(|tx| {
let row = self.batch_row(&tx);
let size = serde_json::to_string(&row).map_or(0, |s| s.len());
((tx, row), size)
});
let mut carry = None;
let mut sent = 0;
loop {
let chunk = fill_chunk(&mut src, &mut carry);
if chunk.is_empty() {
break;
}
let (txs, rows): (Vec<_>, Vec<_>) = chunk.into_iter().unzip();
sent += self.flush_chunk(api_url, api_key, wal, txs, rows);
}
sent
}
fn batch_row(&self, tx: &BufferedTransaction) -> serde_json::Value {
let dashboard_type = dashboard_type(tx.tx_type.as_str());
let job_name = job_name_for(tx);
let (credits_amount, status) = match tx.tx_type.as_str() {
"cancel" => (0.0f64, "failed"),
_ => (tx.amount, "completed"),
};
let mut row = serde_json::json!({
"request_id": tx.request_id,
"zakuro_user_id": tx.user_id,
"job_name": job_name,
"transaction_type": dashboard_type,
"credits_amount": credits_amount,
"status": status,
"duration_ms": if tx.duration_ms > 0.0 { serde_json::Value::Number(serde_json::Number::from_f64(tx.duration_ms).unwrap()) } else { serde_json::Value::Null },
"worker_id": tx.worker_name,
"source_node": tx.source_node,
"compute_hours": if tx.duration_ms > 0.0 { Some(tx.duration_ms / 3_600_000.0) } else { None::<f64> },
"executor": tx.worker_name,
"destination": tx.worker_uri,
"price_per_hour": if tx.price_per_hour > 0.0 { Some(tx.price_per_hour) } else { None::<f64> },
"metadata": self.metadata.get(&tx.request_id).map(|m| m.value().clone())
});
super::ledger::attach_voucher_nonce(&mut row, tx.voucher_nonce.as_deref());
row
}
fn flush_chunk(
&self,
api_url: &str,
api_key: &str,
wal: &Wal,
txs: Vec<BufferedTransaction>,
payload: Vec<serde_json::Value>,
) -> usize {
let tx_count = txs.len();
let endpoint = format!("{}/api/broker/batch-sync", api_url.trim_end_matches('/'));
let payload_str = match serde_json::to_string(&payload) {
Ok(s) => s,
Err(e) => {
eprintln!(" [FLUSH] Failed to serialize payload: {}", e);
for tx in txs {
self.pending.insert(tx.request_id.clone(), tx);
}
return 0;
}
};
match ureq::post(&endpoint)
.config()
.http_status_as_error(false)
.build()
.header("X-Broker-Api-Key", api_key)
.header("Content-Type", "application/json")
.send(payload_str.as_str())
{
Ok(resp) if resp.status().as_u16() == 200 => {
let body = resp.into_body().read_to_string().unwrap_or_default();
let parsed: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
let inserted = parsed["inserted"].as_u64().unwrap_or(tx_count as u64);
let failed = parsed["failed"].as_u64().unwrap_or(0);
let success = parsed["success"].as_bool().unwrap_or(true);
self.total_flushed.fetch_add(inserted, Ordering::Relaxed);
match parsed["results"].as_array() {
Some(results) => {
let mut item_status: std::collections::HashMap<&str, &str> =
std::collections::HashMap::new();
for r in results {
if let (Some(rid), Some(st)) =
(r["request_id"].as_str(), r["status"].as_str())
{
item_status.insert(rid, st);
}
}
let item_error_summary = summarize_item_errors(&parsed["errors"]);
for tx in &txs {
let status = item_status
.get(tx.request_id.as_str())
.copied()
.unwrap_or("applied");
match status {
"failed" => {
self.record_failure(tx.clone(), wal, &item_error_summary);
}
_ => {
self.record_success(&tx.request_id);
retire_delivered(wal, tx);
}
}
}
}
None => {
for tx in &txs {
self.record_success(&tx.request_id);
retire_delivered(wal, tx);
}
}
}
if !success || failed > 0 {
eprintln!(
" [FLUSH] Partial sync to {}: {} inserted, {} failed",
api_url, inserted, failed
);
if let Some(errors) = parsed["errors"].as_array() {
for e in errors.iter().take(3) {
eprintln!(
" [FLUSH] → {}: {}",
e["job_name"].as_str().unwrap_or("?"),
e["error"].as_str().unwrap_or("unknown")
);
}
if errors.len() > 3 {
eprintln!(" [FLUSH] … and {} more errors", errors.len() - 3);
}
}
} else {
println!(
" [FLUSH] Synced {} transactions to {} via API",
inserted, api_url
);
}
inserted as usize
}
Ok(resp) => {
let status = resp.status();
eprintln!(" [FLUSH] API sync failed: status {}", status);
let error = format!("status {}", status);
for tx in txs {
self.record_failure(tx, wal, &error);
}
0
}
Err(e) => {
eprintln!(" [FLUSH] API sync failed: {}", e);
let error = e.to_string();
for tx in txs {
self.record_failure(tx, wal, &error);
}
0
}
}
}
}
impl std::fmt::Debug for TransactionBuffer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TransactionBuffer")
.field("pending", &self.pending.len())
.field("balance_snapshots", &self.balance_snapshots.len())
.field("total_flushed", &self.total_flushed())
.finish()
}
}
#[cfg(test)]
mod flush_batch_size_tests {
use super::*;
fn fat_tx(i: usize) -> BufferedTransaction {
BufferedTransaction {
request_id: format!("earn-{}-aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", i),
user_id: "5492316032".into(),
tx_type: "credit".into(),
amount: 0.012112875187777778,
balance_after: 19998.863504120014,
worker_id: "a59edb75-a8e0-4822-a344-bbffcd791718".into(),
duration_ms: 2340.5812345,
source_node: Some("zc://node-6188100f5daf463e".into()),
worker_name: Some("worker-zc-worker-cloudcompute-larger-name-1".into()),
worker_uri: Some("http://10.13.13.20:3960/execute".into()),
price_per_hour: 3.6,
voucher_nonce: None,
}
}
#[test]
fn a_full_batch_stays_under_the_hub_body_limit() {
let txs: Vec<_> = (0..MAX_FLUSH_BATCH).map(fat_tx).collect();
let payload: Vec<serde_json::Value> = txs
.iter()
.map(|tx| {
serde_json::json!({
"request_id": tx.request_id,
"zakuro_user_id": tx.user_id,
"job_name": job_name_for(tx),
"transaction_type": dashboard_type(tx.tx_type.as_str()),
"credits_amount": tx.amount,
"status": "completed",
"duration_ms": tx.duration_ms,
"worker_id": tx.worker_name,
"source_node": tx.source_node,
"compute_hours": tx.duration_ms / 3_600_000.0,
"executor": tx.worker_name,
"destination": tx.worker_uri,
"price_per_hour": tx.price_per_hour,
"metadata": null
})
})
.collect();
let bytes = serde_json::to_string(&payload).expect("serializes").len();
assert!(
bytes < HUB_MAX_BODY_BYTES / 2,
"a full batch is {bytes} bytes; the hub 413s over {HUB_MAX_BODY_BYTES} \
and the broker sees that as a broken pipe (#330)"
);
}
#[test]
fn chunks_of_metadata_rows_stay_under_the_byte_cap() {
for samples in [crate::sched::bill::MAX_TRUST_SAMPLES, 200] {
let buf = TransactionBuffer::new();
let payloads: Vec<Vec<u8>> = (0..samples as u32)
.map(|i| i.to_le_bytes().to_vec())
.collect();
for i in 0..MAX_FLUSH_BATCH {
let tx = fat_tx(i);
let rid = tx.request_id.clone();
buf.push_transaction(tx);
buf.attach_metadata(
&rid,
crate::sched::bill::trust_metadata(
payloads.iter().map(|p| (p.as_slice(), 123456.0)),
),
);
}
let mut src = buf.pending_snapshot().into_iter().map(|t| {
let row = buf.batch_row(&t);
let size = serde_json::to_string(&row).unwrap().len();
(row, size)
});
let mut carry = None;
let (mut n_chunks, mut covered) = (0, 0);
loop {
let chunk = fill_chunk(&mut src, &mut carry);
if chunk.is_empty() {
break;
}
n_chunks += 1;
covered += chunk.len();
let body = serde_json::to_string(&chunk).unwrap().len();
assert!(
body <= MAX_FLUSH_BYTES && chunk.len() <= MAX_FLUSH_BATCH,
"{samples} samples: a chunk body is {body} bytes / {} rows",
chunk.len()
);
}
assert!(n_chunks > 1, "{samples} samples: must split by bytes");
assert_eq!(covered, MAX_FLUSH_BATCH, "every row lands in a chunk");
}
}
fn chunk_all(sizes: &[usize]) -> Vec<Vec<usize>> {
let mut src = sizes.iter().copied().enumerate();
let mut carry = None;
let mut out = Vec::new();
loop {
let c = fill_chunk(&mut src, &mut carry);
if c.is_empty() {
return out;
}
out.push(c);
}
}
#[test]
fn chunking_edges() {
assert!(chunk_all(&[]).is_empty());
assert_eq!(chunk_all(&[10, 10]), vec![vec![0, 1]]);
let big = MAX_FLUSH_BYTES * 2;
assert_eq!(chunk_all(&[10, big, 10]), vec![vec![0], vec![1], vec![2]]);
assert_eq!(chunk_all(&vec![1; MAX_FLUSH_BATCH + 1]).len(), 2);
}
#[test]
fn at_most_one_chunk_of_rows_is_alive_at_a_time() {
let pulled = std::cell::Cell::new(0usize);
let mut src = (0..50_000usize).map(|i| {
pulled.set(pulled.get() + 1);
(i, 4_000) });
let mut carry = None;
let mut emitted = 0;
let mut chunks = 0;
loop {
let chunk = fill_chunk(&mut src, &mut carry);
if chunk.is_empty() {
break;
}
emitted += chunk.len();
chunks += 1;
assert!(chunk.len() < MAX_FLUSH_BATCH);
assert!(
pulled.get() <= emitted + 1,
"pulled {} rows but only {emitted} are in sent chunks",
pulled.get()
);
}
assert_eq!(emitted, 50_000);
assert!(chunks > 300);
}
#[test]
fn a_full_quarantine_file_is_rotated_not_grown_forever() {
let dir = std::env::temp_dir().join(format!("zc-quarantine-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("tmpdir");
let path = dir.join("wal-rejected.jsonl");
std::fs::write(&path, b"small\n").expect("write");
TransactionBuffer::rotate_quarantine_if_full(&path);
assert!(path.exists(), "a small file is not rotated");
assert!(
!dir.join("wal-rejected.jsonl.1").exists(),
"nothing rotated yet"
);
let big = vec![b'x'; TransactionBuffer::MAX_QUARANTINE_BYTES as usize + 1];
std::fs::write(&path, &big).expect("write big");
TransactionBuffer::rotate_quarantine_if_full(&path);
assert!(!path.exists(), "the full file was moved aside");
let rotated = dir.join("wal-rejected.jsonl.1");
assert!(rotated.exists(), "one previous generation is kept");
assert_eq!(
std::fs::metadata(&rotated).expect("rotated").len(),
big.len() as u64,
"the rotated file keeps its contents"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_backlog_is_split_into_bounded_requests() {
let backlog = 274_461; let chunks = (0..backlog)
.collect::<Vec<_>>()
.chunks(MAX_FLUSH_BATCH)
.len();
assert!(chunks > 1, "a backlog must not be one request");
assert!(
chunks * MAX_FLUSH_BATCH >= backlog,
"every transaction must land in some chunk"
);
}
}
#[cfg(test)]
mod earn_credit_purchase_payload_tests {
use super::*;
fn capturing_hub() -> (String, std::sync::mpsc::Receiver<String>) {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
let _ = tx.send(body);
let _ = req.respond(tiny_http::Response::from_string(
r#"{"success":true,"inserted":1,"failed":0}"#,
));
}
});
(format!("http://127.0.0.1:{port}"), rx)
}
#[test]
fn flush_sends_a_credit_transaction_as_credit_purchase() {
let path = format!(
"/tmp/zc_test_flush_dashboard_type_{}_{}.jsonl",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
);
let _ = std::fs::remove_file(&path);
let wal = Wal::open(&path).expect("open test WAL");
let buf = TransactionBuffer::new();
buf.push_transaction(BufferedTransaction {
request_id: "earn-r1".to_string(),
user_id: "owner".to_string(),
tx_type: "credit".to_string(),
amount: 1.5,
balance_after: 1.5,
worker_id: "w0".to_string(),
duration_ms: 10.0,
source_node: None,
worker_name: Some("w0".to_string()),
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
});
let (hub, bodies) = capturing_hub();
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 1);
let sent: serde_json::Value = serde_json::from_str(&bodies.recv().unwrap()).unwrap();
assert_eq!(
sent[0]["request_id"], "earn-r1",
"the earn row must be the one sent"
);
assert_eq!(
sent[0]["transaction_type"], "credit_purchase",
"a `credit` tx must be sent to the hub as `credit_purchase`, or the \
earnings summary silently drops it"
);
let _ = std::fs::remove_file(&path);
}
}
#[cfg(test)]
mod quarantine_tests {
use super::*;
fn test_wal(name: &str) -> (std::path::PathBuf, Wal) {
let path = std::path::PathBuf::from(format!(
"/tmp/zc_test_flush_quarantine_{}_{}_{}.jsonl",
name,
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_file(&path);
let wal = Wal::open(&path).expect("open test WAL");
(path, wal)
}
fn cleanup(wal_path: &std::path::Path, wal: &Wal) {
let _ = std::fs::remove_file(wal_path);
let _ = std::fs::remove_file(TransactionBuffer::rejected_path(wal));
}
fn tx(request_id: &str, user_id: &str) -> BufferedTransaction {
BufferedTransaction {
request_id: request_id.to_string(),
user_id: user_id.to_string(),
tx_type: "commit".to_string(),
amount: 1.0,
balance_after: 1.0,
worker_id: "w0".to_string(),
duration_ms: 10.0,
source_node: None,
worker_name: Some("w0".to_string()),
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
}
}
fn read_rejected_lines(wal: &Wal) -> Vec<serde_json::Value> {
let path = TransactionBuffer::rejected_path(wal);
match std::fs::read_to_string(&path) {
Ok(contents) => contents
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| serde_json::from_str(l).expect("quarantine line must be valid JSON"))
.collect(),
Err(_) => Vec::new(),
}
}
fn hub_always(status: u16) -> String {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
std::thread::spawn(move || {
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
let _ = req.respond(tiny_http::Response::from_string("").with_status_code(status));
}
});
format!("http://127.0.0.1:{port}")
}
fn dead_hub() -> String {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
drop(server); format!("http://127.0.0.1:{port}")
}
fn hub_fail_then_succeed(fail_times: usize, status: u16) -> String {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
std::thread::spawn(move || {
let mut seen = 0usize;
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
seen += 1;
if seen <= fail_times {
let _ =
req.respond(tiny_http::Response::from_string("").with_status_code(status));
} else {
let _ = req.respond(tiny_http::Response::from_string(
r#"{"success":true,"inserted":1,"failed":0}"#,
));
}
}
});
format!("http://127.0.0.1:{port}")
}
fn hub_with_item_failure(bad_rid: &'static str) -> String {
hub_with_item_failure_then_succeed(bad_rid, usize::MAX)
}
fn hub_with_item_failure_then_succeed(bad_rid: &'static str, fail_times: usize) -> String {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
std::thread::spawn(move || {
let mut seen = 0usize;
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
seen += 1;
let bad_still_fails = seen <= fail_times;
let parsed: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
let items = parsed.as_array().cloned().unwrap_or_default();
let results: Vec<serde_json::Value> = items
.iter()
.map(|item| {
let rid = item["request_id"].as_str().unwrap_or("");
let status = if rid == bad_rid && bad_still_fails {
"failed"
} else {
"applied"
};
serde_json::json!({"request_id": rid, "status": status})
})
.collect();
let failed = results.iter().filter(|r| r["status"] == "failed").count();
let inserted = results.len() - failed;
let resp_body = serde_json::json!({
"success": failed == 0,
"inserted": inserted,
"failed": failed,
"results": results,
"errors": if failed > 0 {
serde_json::json!([{"job_name": "Compute Job (w0)", "error": "User not found: anonymous"}])
} else {
serde_json::json!([])
},
})
.to_string();
let _ = req.respond(tiny_http::Response::from_string(resp_body));
}
});
format!("http://127.0.0.1:{port}")
}
fn executed(request_id: &str) -> crate::broker::wal::WalEntry {
crate::broker::wal::WalEntry {
request_id: request_id.to_string(),
user_id: "alice".to_string(),
reservation_id: format!("res-{request_id}"),
estimated_cost: 1.0,
actual_cost: Some(1.0),
worker_id: "w0".to_string(),
worker_name: Some("w0".to_string()),
duration_ms: Some(10.0),
timestamp: chrono::Utc::now(),
status: WalStatus::Executed,
voucher_nonce: None,
}
}
fn still_owed(wal: &Wal, request_id: &str) -> bool {
wal.read_uncommitted()
.unwrap()
.iter()
.any(|e| e.request_id == request_id)
}
#[test]
fn a_delivered_debit_retires_its_wal_entry() {
let (path, wal) = test_wal("debit_retired");
wal.append(&executed("r-debit")).unwrap();
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-debit", "alice"));
assert!(still_owed(&wal, "r-debit"), "owed until delivered");
let hub = hub_with_item_failure("nobody");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 1);
assert!(
!still_owed(&wal, "r-debit"),
"delivered, so no longer replayed"
);
cleanup(&path, &wal);
}
#[test]
fn a_delivered_debit_retires_its_wal_entry_on_an_older_hub() {
let (path, wal) = test_wal("debit_retired_legacy");
wal.append(&executed("r-old")).unwrap();
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-old", "alice"));
let hub = hub_fail_then_succeed(0, 200);
buf.flush_to_api(&hub, "key", &wal);
assert!(!still_owed(&wal, "r-old"));
cleanup(&path, &wal);
}
#[test]
fn a_rejected_debit_stays_owed() {
let (path, wal) = test_wal("debit_rejected");
wal.append(&executed("r-bad")).unwrap();
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-bad", "alice"));
let hub = hub_with_item_failure("r-bad");
buf.flush_to_api(&hub, "key", &wal);
assert!(still_owed(&wal, "r-bad"));
cleanup(&path, &wal);
}
#[test]
fn whole_request_failure_quarantines_after_max_attempts() {
let (path, wal) = test_wal("always_500");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-bad", "alice"));
let hub = hub_always(500);
for _ in 0..MAX_FLUSH_ATTEMPTS {
assert_eq!(buf.pending_count(), 1, "not yet quarantined");
let flushed = buf.flush_to_api(&hub, "key", &wal);
assert_eq!(flushed, 0);
}
assert_eq!(buf.pending_count(), 0, "must be gone from pending");
let rejected = read_rejected_lines(&wal);
assert_eq!(
rejected.len(),
1,
"the quarantine event (and its one warning) must fire exactly once, \
not once per attempt"
);
assert_eq!(rejected[0]["request_id"], "r-bad");
cleanup(&path, &wal);
}
#[test]
fn quarantined_entry_does_not_block_later_transactions() {
let (path, wal) = test_wal("unblocks_queue");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-bad", "alice"));
let bad_hub = hub_always(500);
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&bad_hub, "key", &wal);
}
assert_eq!(buf.pending_count(), 0, "bad tx quarantined out of pending");
buf.push_transaction(tx("r-good", "bob"));
let good_hub_url = {
let (server, port) = crate::integration_tests::integration::bind_loopback_http();
std::thread::spawn(move || {
for mut req in server.incoming_requests() {
let mut body = String::new();
let _ = std::io::Read::read_to_string(req.as_reader(), &mut body);
let _ = req.respond(tiny_http::Response::from_string(
r#"{"success":true,"inserted":1,"failed":0}"#,
));
}
});
format!("http://127.0.0.1:{port}")
};
let flushed = buf.flush_to_api(&good_hub_url, "key", &wal);
assert_eq!(flushed, 1, "the later transaction must reach the hub");
assert_eq!(buf.pending_count(), 0);
cleanup(&path, &wal);
}
#[test]
fn transport_error_counts_as_an_attempt_and_quarantines() {
let (path, wal) = test_wal("transport_error");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-unreachable", "alice"));
let hub = dead_hub();
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key", &wal);
}
assert_eq!(buf.pending_count(), 0);
let rejected = read_rejected_lines(&wal);
assert_eq!(rejected.len(), 1);
assert_eq!(rejected[0]["request_id"], "r-unreachable");
cleanup(&path, &wal);
}
#[test]
fn per_item_rejection_inside_200_gets_the_same_attempt_grace() {
let (path, wal) = test_wal("per_item");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-bad-item", "anonymous-ish"));
buf.push_transaction(tx("r-good-item", "bob"));
let hub = hub_with_item_failure("r-bad-item");
for i in 0..MAX_FLUSH_ATTEMPTS {
let flushed = buf.flush_to_api(&hub, "key", &wal);
if i == 0 {
assert_eq!(
flushed, 1,
"the good item in the same batch must still flush on the first attempt"
);
}
}
assert_eq!(buf.pending_count(), 0, "must be gone from pending");
let rejected = read_rejected_lines(&wal);
assert_eq!(
rejected.len(),
1,
"quarantined exactly once, after MAX_FLUSH_ATTEMPTS, not on the first failure"
);
assert_eq!(rejected[0]["request_id"], "r-bad-item");
assert!(
rejected[0]["error"]
.as_str()
.unwrap_or("")
.contains("User not found"),
"the hub's per-item error text must be preserved"
);
cleanup(&path, &wal);
}
#[test]
fn per_item_rejection_that_later_succeeds_is_not_quarantined() {
let (path, wal) = test_wal("per_item_transient");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-transient-item", "bob"));
let hub = hub_with_item_failure_then_succeed("r-transient-item", 2);
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 0); assert_eq!(buf.pending_count(), 1, "requeued, not quarantined");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 0); assert_eq!(buf.pending_count(), 1, "requeued, not quarantined");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 1);
assert_eq!(buf.pending_count(), 0);
assert!(
read_rejected_lines(&wal).is_empty(),
"a per-item failure that eventually succeeds must never be quarantined"
);
cleanup(&path, &wal);
}
#[test]
fn transient_failure_then_success_is_not_quarantined() {
let (path, wal) = test_wal("transient");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-transient", "alice"));
let hub = hub_fail_then_succeed(2, 500);
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 0); assert_eq!(buf.pending_count(), 1, "requeued, not quarantined");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 0); assert_eq!(buf.pending_count(), 1, "requeued, not quarantined");
assert_eq!(buf.flush_to_api(&hub, "key", &wal), 1);
assert_eq!(buf.pending_count(), 0);
assert!(
read_rejected_lines(&wal).is_empty(),
"a transaction that eventually succeeds must never be quarantined"
);
cleanup(&path, &wal);
}
#[test]
fn anonymous_sentinel_is_never_buffered() {
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-anon", ANONYMOUS_USER));
assert_eq!(
buf.pending_count(),
0,
"the anonymous sentinel must never be buffered for the hub"
);
buf.push_transaction(tx("r-real", "alice"));
assert_eq!(
buf.pending_count(),
1,
"a normal user must still be buffered"
);
}
#[test]
fn quarantine_file_lines_are_valid_json_with_error_text() {
let (path, wal) = test_wal("valid_json_lines");
let buf = TransactionBuffer::new();
buf.push_transaction(tx("r-one", "alice"));
buf.push_transaction(tx("r-two", "alice"));
let hub = hub_always(500);
for _ in 0..MAX_FLUSH_ATTEMPTS {
buf.flush_to_api(&hub, "key", &wal);
}
let rejected_path = TransactionBuffer::rejected_path(&wal);
let contents = std::fs::read_to_string(&rejected_path).expect("quarantine file exists");
let lines: Vec<&str> = contents.lines().filter(|l| !l.trim().is_empty()).collect();
assert_eq!(lines.len(), 2, "one line per quarantined transaction");
for line in &lines {
let parsed: serde_json::Value =
serde_json::from_str(line).expect("each line must be standalone valid JSON");
assert!(
parsed["error"].as_str().is_some_and(|s| !s.is_empty()),
"each quarantined line must carry the error text"
);
}
cleanup(&path, &wal);
}
}
#[cfg(test)]
mod voucher_nonce_tests {
use super::*;
fn tx(nonce: Option<&str>) -> BufferedTransaction {
BufferedTransaction {
request_id: "r".into(),
user_id: "u".into(),
tx_type: "commit".into(),
amount: 1.0,
balance_after: 9.0,
worker_id: "w".into(),
duration_ms: 10.0,
source_node: None,
worker_name: Some("w".into()),
worker_uri: None,
price_per_hour: 1.0,
voucher_nonce: nonce.map(String::from),
}
}
#[test]
fn batch_row_carries_the_nonce_only_when_present() {
let buf = TransactionBuffer::new();
let row = buf.batch_row(&tx(Some("n-1")));
assert_eq!(row["voucher_nonce"], "n-1");
assert!(buf.batch_row(&tx(None)).get("voucher_nonce").is_none());
assert!(buf.batch_row(&tx(Some(""))).get("voucher_nonce").is_none());
}
#[test]
fn buffered_transaction_round_trips_and_old_json_decodes() {
let back: BufferedTransaction =
serde_json::from_str(&serde_json::to_string(&tx(Some("n-2"))).unwrap()).unwrap();
assert_eq!(back.voucher_nonce.as_deref(), Some("n-2"));
let plain = serde_json::to_string(&tx(None)).unwrap();
assert!(!plain.contains("voucher_nonce"));
let old = r#"{"request_id":"r","user_id":"u","tx_type":"commit","amount":1.0,"balance_after":2.0,"worker_id":"w","duration_ms":1.0,"source_node":null,"worker_name":null,"worker_uri":null,"price_per_hour":0.0}"#;
let back: BufferedTransaction = serde_json::from_str(old).unwrap();
assert_eq!(back.voucher_nonce, None);
}
}