use super::flush::{BufferedTransaction, TransactionBuffer};
use super::ledger::Ledger;
use super::wal::{PayoutTo, Wal, WalEntry, WalPayout, WalStatus};
pub fn owed_payout_id(request_id: &str) -> String {
format!("peer-earn-{request_id}")
}
pub fn replay_wal(
wal: &Wal,
ledger: &Ledger,
tx_buffer: &TransactionBuffer,
node_name: Option<&str>,
) -> Vec<WalEntry> {
let mut owed = Vec::new();
let uncommitted = match wal.read_uncommitted() {
Ok(entries) => entries,
Err(e) => {
eprintln!(" [WAL] Failed to read uncommitted entries: {}", e);
return owed;
}
};
if uncommitted.is_empty() {
return owed;
}
println!(
" [WAL] Replaying {} uncommitted entries...",
uncommitted.len()
);
for entry in &uncommitted {
match entry.status {
WalStatus::Reserved => {
println!(
" [WAL] Closing stale reservation for user {} (req: {}, held: {:.6}); nothing to refund",
entry.user_id, entry.request_id, entry.estimated_cost
);
let _ = wal.update_status(&entry.request_id, WalStatus::Failed, None, None);
}
WalStatus::Executed | WalStatus::Earned | WalStatus::RefundOwed if entry.hubless => {
let _ = wal.update_status(&entry.request_id, WalStatus::Committed, None, None);
}
WalStatus::EarnOwed if entry.hubless => {
let _ = wal.update_status(&entry.request_id, WalStatus::Failed, None, None);
}
WalStatus::Executed
| WalStatus::Earned
| WalStatus::EarnOwed
| WalStatus::RefundOwed
if !ledger.has_hub() =>
{
eprintln!(
" [WAL] {} ({:?}) is owed to a hub this broker has none of; kept as is",
entry.request_id, entry.status
);
}
WalStatus::Executed => {
let actual_cost = entry.actual_cost.unwrap_or(entry.estimated_cost);
println!(
" [WAL] Re-delivering unflushed debit {} for user {} (cost: {:.6})",
entry.request_id, entry.user_id, actual_cost
);
let balance_after = ledger
.get_authoritative_balance(&entry.user_id)
.unwrap_or(0.0);
let worker_identifier = entry
.worker_name
.clone()
.unwrap_or_else(|| entry.worker_id.clone());
tx_buffer.push_transaction(BufferedTransaction {
request_id: entry.request_id.clone(),
user_id: entry.user_id.clone(),
tx_type: "commit".to_string(),
amount: actual_cost,
balance_after,
worker_id: entry.worker_id.clone(),
duration_ms: entry.duration_ms.unwrap_or(0.0),
source_node: node_name.map(|s| s.to_string()),
worker_name: Some(worker_identifier),
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: entry.voucher_nonce.clone(),
});
if let Some(payout) = &entry.payout {
if let Some(o) = replay_payout(wal, ledger, tx_buffer, node_name, entry, payout)
{
owed.push(o);
}
}
}
WalStatus::Earned => {
rebuffer_earn(ledger, tx_buffer, node_name, entry);
}
WalStatus::EarnOwed => owed.push(entry.clone()),
WalStatus::RefundOwed => rebuffer_refund(ledger, tx_buffer, node_name, entry),
_ => {}
}
}
if let Err(e) = wal.compact() {
eprintln!(" [WAL] Failed to compact after replay: {}", e);
}
println!(" [WAL] Replay complete");
owed
}
fn rebuffer_earn(
ledger: &Ledger,
tx_buffer: &TransactionBuffer,
node_name: Option<&str>,
entry: &WalEntry,
) {
let amount = entry.actual_cost.unwrap_or(entry.estimated_cost);
println!(
" [WAL] Re-delivering unflushed earn {} for user {} (amount: {:.6})",
entry.request_id, entry.user_id, amount
);
let balance_after = ledger
.get_authoritative_balance(&entry.user_id)
.unwrap_or(0.0);
tx_buffer.push_transaction(BufferedTransaction {
request_id: entry.request_id.clone(),
user_id: entry.user_id.clone(),
tx_type: "earning".to_string(),
amount,
balance_after,
worker_id: entry.worker_id.clone(),
duration_ms: entry.duration_ms.unwrap_or(0.0),
source_node: node_name.map(|s| s.to_string()),
worker_name: entry
.worker_name
.clone()
.or_else(|| Some(entry.worker_id.clone())),
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
});
}
fn replay_payout(
wal: &Wal,
ledger: &Ledger,
tx_buffer: &TransactionBuffer,
node_name: Option<&str>,
job: &WalEntry,
payout: &WalPayout,
) -> Option<WalEntry> {
if payout.amount <= 0.0 {
return None;
}
let worker_name = job
.worker_name
.clone()
.unwrap_or_else(|| job.worker_id.clone());
match &payout.to {
PayoutTo::Owner { user_id } => {
let earned = WalEntry {
request_id: payout.row_id.clone(),
user_id: user_id.clone(),
reservation_id: String::new(),
estimated_cost: payout.amount,
actual_cost: Some(payout.amount),
worker_id: job.worker_id.clone(),
worker_name: Some(worker_name),
duration_ms: job.duration_ms,
timestamp: chrono::Utc::now(),
status: WalStatus::Earned,
voucher_nonce: None,
unit: super::wal::WAL_UNIT.to_string(),
payout: None,
hubless: false,
receipt_hash: None,
};
if matches!(wal.append_new(&earned), Ok(true)) {
rebuffer_earn(ledger, tx_buffer, node_name, &earned);
}
None
}
PayoutTo::Requester => {
let refund = refund_owed_entry(
&payout.row_id,
&job.user_id,
payout.amount,
&job.worker_id,
&worker_name,
job.duration_ms,
);
if matches!(wal.append_new(&refund), Ok(true)) {
rebuffer_refund(ledger, tx_buffer, node_name, &refund);
}
None
}
PayoutTo::Peer { .. } => {
let owed = owed_payout_entry(job, &worker_name, payout.clone());
if !matches!(wal.append_new(&owed), Ok(true)) {
return None;
}
println!(
" [WAL] Payout {:.6} for {} owed to its remote executor",
payout.amount, job.request_id
);
Some(owed)
}
}
}
pub fn refund_owed_entry(
row_id: &str,
user_id: &str,
amount: f64,
worker_id: &str,
worker_name: &str,
duration_ms: Option<f64>,
) -> WalEntry {
WalEntry {
request_id: row_id.to_string(),
user_id: user_id.to_string(),
reservation_id: String::new(),
estimated_cost: amount,
actual_cost: Some(amount),
worker_id: worker_id.to_string(),
worker_name: Some(worker_name.to_string()),
duration_ms,
timestamp: chrono::Utc::now(),
status: WalStatus::RefundOwed,
voucher_nonce: None,
unit: super::wal::WAL_UNIT.to_string(),
payout: None,
hubless: false,
receipt_hash: None,
}
}
fn rebuffer_refund(
ledger: &Ledger,
tx_buffer: &TransactionBuffer,
node_name: Option<&str>,
entry: &WalEntry,
) {
let amount = entry.actual_cost.unwrap_or(entry.estimated_cost);
println!(
" [WAL] Re-delivering refund {} for user {} (amount: {:.6})",
entry.request_id, entry.user_id, amount
);
tx_buffer.push_transaction(BufferedTransaction {
request_id: entry.request_id.clone(),
user_id: entry.user_id.clone(),
tx_type: "refund".to_string(),
amount,
balance_after: ledger
.get_authoritative_balance(&entry.user_id)
.unwrap_or(0.0),
worker_id: entry.worker_id.clone(),
duration_ms: entry.duration_ms.unwrap_or(0.0),
source_node: node_name.map(|s| s.to_string()),
worker_name: entry
.worker_name
.clone()
.or_else(|| Some(entry.worker_id.clone())),
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
});
}
pub fn owed_payout_entry(job: &WalEntry, worker_name: &str, payout: WalPayout) -> WalEntry {
WalEntry {
request_id: owed_payout_id(&job.request_id),
user_id: job.user_id.clone(),
reservation_id: String::new(),
estimated_cost: payout.amount,
actual_cost: Some(payout.amount),
worker_id: job.worker_id.clone(),
worker_name: Some(worker_name.to_string()),
duration_ms: job.duration_ms,
timestamp: chrono::Utc::now(),
status: WalStatus::EarnOwed,
voucher_nonce: None,
unit: super::wal::WAL_UNIT.to_string(),
payout: Some(payout),
hubless: false,
receipt_hash: None,
}
}
#[cfg(test)]
mod worker_attribution_tests {
use super::*;
use crate::broker::wal::WalEntry;
use std::sync::{Arc, Mutex};
fn tmp_wal(name: &str) -> std::path::PathBuf {
std::path::PathBuf::from(format!("/tmp/zc_recovery_test_{}.jsonl", name))
}
fn cleanup(p: &std::path::Path) {
let _ = std::fs::remove_file(p);
let _ = std::fs::remove_file(p.with_extension("tmp"));
}
fn capturing_hub() -> (String, Arc<Mutex<Option<String>>>) {
let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
let port = server.server_addr().to_ip().unwrap().port();
let captured = Arc::new(Mutex::new(None));
let seen = captured.clone();
std::thread::spawn(move || {
while let Ok(Some(mut req)) = server.recv_timeout(std::time::Duration::from_secs(5)) {
let mut body = String::new();
let _ = req.as_reader().read_to_string(&mut body);
if req.url().ends_with("/api/broker/batch-sync") {
*seen.lock().unwrap() = Some(body);
}
let _ = req.respond(tiny_http::Response::from_string("{}").with_status_code(200));
}
});
(format!("http://127.0.0.1:{port}"), captured)
}
fn replay_bounded(wal: Wal, hub: String, tx_buffer: TransactionBuffer) {
let (done_tx, done_rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let ledger = Ledger::new(Some(hub.clone()), Some("zk_1_test".to_string()));
replay_wal(&wal, &ledger, &tx_buffer, None);
tx_buffer.flush_to_api(&hub, "zk_1_test", &wal);
let _ = done_tx.send(());
});
done_rx
.recv_timeout(std::time::Duration::from_secs(30))
.expect("replay_wal did not return within 30s (blocked on a hub call?)");
}
fn wait_for_json(captured: &Arc<Mutex<Option<String>>>) -> serde_json::Value {
for _ in 0..300 {
if let Some(body) = captured.lock().unwrap().clone() {
let rows: serde_json::Value = serde_json::from_str(&body)
.unwrap_or_else(|e| panic!("captured body {body:?} was not JSON: {e}"));
return rows[0].clone();
}
std::thread::sleep(std::time::Duration::from_millis(10));
}
panic!("the mock hub never received the dashboard POST");
}
fn executed_entry(request_id: &str, worker_id: &str, worker_name: Option<&str>) -> WalEntry {
WalEntry {
request_id: request_id.to_string(),
user_id: "alice".to_string(),
reservation_id: format!("res-{request_id}"),
estimated_cost: 0.05,
actual_cost: Some(0.05),
worker_id: worker_id.to_string(),
worker_name: worker_name.map(str::to_string),
duration_ms: Some(1000.0),
timestamp: chrono::Utc::now(),
status: WalStatus::Executed,
voucher_nonce: None,
unit: crate::broker::wal::WAL_UNIT.to_string(),
payout: None,
hubless: false,
receipt_hash: None,
}
}
#[test]
fn replay_of_an_executed_entry_publishes_the_workers_recorded_name() {
let path = tmp_wal("executed_with_name");
cleanup(&path);
let wal = Wal::open(&path).unwrap();
wal.append(&executed_entry(
"req-exec-1",
"ephemeral-uuid-1234",
Some("gpu-box-1"),
))
.unwrap();
wal.flush_buffer().unwrap();
let (hub, captured) = capturing_hub();
let tx_buffer = TransactionBuffer::new();
replay_bounded(wal, hub, tx_buffer);
let body = wait_for_json(&captured);
assert_eq!(
body["worker_id"], "gpu-box-1",
"replay must publish the worker's registered name, body: {body}"
);
assert_ne!(
body["worker_id"], "ephemeral-uuid-1234",
"must not leak the ephemeral registry id to the hub, body: {body}"
);
assert_eq!(body["request_id"], "req-exec-1", "body: {body}");
cleanup(&path);
}
#[test]
fn replay_of_an_executed_entry_without_a_recorded_name_falls_back_to_worker_id() {
let path = tmp_wal("executed_without_name");
cleanup(&path);
let wal = Wal::open(&path).unwrap();
wal.append(&executed_entry("req-exec-2", "legacy-worker-id", None))
.unwrap();
wal.flush_buffer().unwrap();
let (hub, captured) = capturing_hub();
let tx_buffer = TransactionBuffer::new();
replay_bounded(wal, hub, tx_buffer);
let body = wait_for_json(&captured);
assert_eq!(body["worker_id"], "legacy-worker-id", "body: {body}");
cleanup(&path);
}
#[test]
fn replay_of_an_earned_entry_rebuffers_the_workers_recorded_name() {
let path = tmp_wal("earned_with_name");
cleanup(&path);
let wal = Wal::open(&path).unwrap();
wal.append(&WalEntry {
request_id: "earn-req-1".to_string(),
user_id: "owner".to_string(),
reservation_id: String::new(),
estimated_cost: 1.0,
actual_cost: Some(1.0),
worker_id: "ephemeral-uuid-5678".to_string(),
worker_name: Some("gpu-box-2".to_string()),
duration_ms: Some(500.0),
timestamp: chrono::Utc::now(),
status: WalStatus::Earned,
voucher_nonce: None,
unit: crate::broker::wal::WAL_UNIT.to_string(),
payout: None,
hubless: false,
receipt_hash: None,
})
.unwrap();
wal.flush_buffer().unwrap();
let ledger = Ledger::new(Some("http://127.0.0.1:1".into()), Some("k".into()));
let tx_buffer = TransactionBuffer::new();
replay_wal(&wal, &ledger, &tx_buffer, None);
let queued = tx_buffer.pending_snapshot();
let tx = queued
.iter()
.find(|t| t.request_id == "earn-req-1")
.expect("the unflushed earn must be re-buffered");
assert_eq!(
tx.worker_name,
Some("gpu-box-2".to_string()),
"re-buffered earn must carry the worker's recorded name, not its ephemeral id"
);
cleanup(&path);
}
}
#[cfg(test)]
mod test_hub {
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::Duration;
#[derive(Clone, Copy)]
pub(super) enum Mode {
Accept,
ItemError(&'static str),
Status(u16),
InvalidRow(&'static str),
InvalidSuffix(&'static str),
}
pub(super) struct BookHub {
pub(super) url: String,
book: Arc<Mutex<HashMap<String, (String, f64)>>>,
pub(super) received: Arc<Mutex<HashMap<String, usize>>>,
requests: Arc<std::sync::atomic::AtomicUsize>,
}
impl BookHub {
pub(super) fn start() -> Self {
Self::with(Mode::Accept)
}
pub(super) fn with(mode: Mode) -> Self {
let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
let url = format!(
"http://127.0.0.1:{}",
server.server_addr().to_ip().unwrap().port()
);
let book = Arc::new(Mutex::new(HashMap::new()));
let received = Arc::new(Mutex::new(HashMap::new()));
let requests = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let (b, n, q) = (book.clone(), received.clone(), requests.clone());
std::thread::spawn(move || {
while let Ok(Some(mut req)) = server.recv_timeout(Duration::from_secs(10)) {
let mut body = String::new();
let _ = req.as_reader().read_to_string(&mut body);
if req.url().ends_with("/api/broker/batch-sync") {
q.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
let rows = match serde_json::from_str::<serde_json::Value>(&body) {
Ok(serde_json::Value::Array(rows)) => rows,
Ok(row) if req.url().ends_with("/api/broker/transaction") => vec![row],
_ => Vec::new(),
};
let rids: Vec<String> = rows
.iter()
.filter_map(|r| r["request_id"].as_str().map(str::to_string))
.collect();
for rid in &rids {
*n.lock().unwrap().entry(rid.clone()).or_default() += 1;
}
let (status, answer) = match mode {
Mode::Status(code) => (code, "{}".to_string()),
Mode::InvalidRow(bad) if rids.iter().any(|r| r == bad) => {
(422, r#"{"detail":"validation error"}"#.to_string())
}
Mode::InvalidSuffix(bad) if rids.iter().any(|r| r.ends_with(bad)) => {
(422, r#"{"detail":"validation error"}"#.to_string())
}
Mode::ItemError(error) if !rows.is_empty() => {
let results: Vec<_> = rids
.iter()
.map(|r| {
serde_json::json!({
"request_id": r, "status": "failed", "error": error,
})
})
.collect();
let detail = serde_json::json!({
"results": results, "success": false, "inserted": 0,
"failed": rids.len(),
});
(500, serde_json::json!({ "detail": detail }).to_string())
}
_ => {
for row in &rows {
if let Some(rid) = row["request_id"].as_str() {
b.lock().unwrap().entry(rid.to_string()).or_insert((
row["transaction_type"].as_str().unwrap_or("").to_string(),
row["amount"].as_f64().unwrap_or(f64::NAN),
));
}
}
(200, "{}".to_string())
}
};
let _ = req
.respond(tiny_http::Response::from_string(answer).with_status_code(status));
}
});
Self {
url,
book,
received,
requests,
}
}
pub(super) fn requests(&self) -> usize {
self.requests.load(std::sync::atomic::Ordering::SeqCst)
}
pub(super) fn booked(&self, rid: &str, kind: &str) -> Option<f64> {
self.book.lock().unwrap().get(rid).map(|(t, a)| {
assert_eq!(t, kind, "{rid} booked as {t}");
*a
})
}
pub(super) fn received(&self, rid: &str) -> usize {
self.received.lock().unwrap().get(rid).copied().unwrap_or(0)
}
}
}
#[cfg(test)]
mod replay_retirement_tests {
use super::test_hub::{BookHub, Mode};
use super::*;
use crate::broker::wal::WalEntry;
use std::path::{Path, PathBuf};
const DOWN: &str = "http://127.0.0.1:1";
fn wal_dir(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("zc-replay-{name}-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
dir
}
fn executed(rid: &str) -> WalEntry {
WalEntry {
request_id: rid.to_string(),
user_id: "u1".to_string(),
reservation_id: format!("res-{rid}"),
estimated_cost: 10.0,
actual_cost: Some(4.0),
worker_id: "wid".to_string(),
worker_name: Some("w".to_string()),
duration_ms: Some(1000.0),
timestamp: chrono::Utc::now(),
status: WalStatus::Executed,
voucher_nonce: None,
unit: crate::broker::wal::WAL_UNIT.to_string(),
payout: None,
hubless: false,
receipt_hash: None,
}
}
fn write_wal(dir: &Path, entries: &[WalEntry]) {
let wal = Wal::open(dir.join("wal.jsonl")).unwrap();
for e in entries {
wal.append(e).unwrap();
}
wal.flush_buffer().unwrap();
}
fn restart(dir: &Path, hub_url: Option<&str>, flushes: usize) -> (Ledger, Vec<WalEntry>) {
let wal = Wal::open(dir.join("wal.jsonl")).unwrap();
let ledger = Ledger::new(
hub_url.map(str::to_string),
hub_url.map(|_| "k".to_string()),
);
let tx_buffer = TransactionBuffer::new();
replay_wal(&wal, &ledger, &tx_buffer, None);
if let Some(url) = hub_url {
for _ in 0..flushes {
if tx_buffer.pending_count() == 0 {
break;
}
tx_buffer.flush_to_api(url, "k", &wal);
}
}
wal.flush_buffer().unwrap();
let owed = wal.read_uncommitted().unwrap();
(ledger, owed)
}
fn local_balance(ledger: &Ledger, user: &str) -> f64 {
ledger.local_balances.get(user).map_or(0.0, |b| *b)
}
#[test]
fn an_undelivered_debit_is_delivered_once_and_never_refunded_on_restart() {
let dir = wal_dir("undelivered");
write_wal(&dir, &[executed("job-a")]);
let (ledger, owed) = restart(&dir, Some(DOWN), 1);
assert!(owed.iter().any(|e| e.request_id == "job-a"), "{owed:?}");
assert!(
local_balance(&ledger, "u1").abs() < 1e-9,
"no refund minted"
);
let hub = BookHub::start();
for _ in 0..2 {
let (ledger, owed) = restart(&dir, Some(&hub.url), 1);
assert!(owed.is_empty(), "delivered, so retired: {owed:?}");
assert!(
local_balance(&ledger, "u1").abs() < 1e-9,
"no refund minted"
);
}
assert_eq!(hub.received("job-a"), 1);
assert_eq!(hub.booked("job-a", "job_execution"), Some(4.0));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_permanently_refused_debit_is_quarantined_once_not_on_every_restart() {
let dir = wal_dir("refused");
write_wal(&dir, &[executed("job-r")]);
let hub = BookHub::with(Mode::ItemError("User not found: u1"));
let (_, owed) = restart(&dir, Some(&hub.url), 20);
assert!(owed.is_empty(), "quarantined, so retired: {owed:?}");
let attempts = hub.received("job-r");
assert!(attempts > 0);
let (_, owed) = restart(&dir, Some(&hub.url), 20);
assert!(owed.is_empty(), "{owed:?}");
assert_eq!(hub.received("job-r"), attempts, "re-sent after quarantine");
let rejected = std::fs::read_to_string(dir.join("wal-rejected.jsonl")).unwrap();
assert_eq!(
rejected.lines().filter(|l| l.contains("job-r")).count(),
1,
"{rejected}"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_debit_refused_for_a_reason_that_clears_is_delivered_after_a_restart() {
for mode in [
Mode::Status(400),
Mode::Status(404),
Mode::Status(403),
Mode::Status(429),
Mode::ItemError("403: Acting for another account requires a hub voucher for that account (lookup_error)"),
] {
let dir = wal_dir("transient");
write_wal(&dir, &[executed("job-t")]);
let refusing = BookHub::with(mode);
let (_, owed) = restart(&dir, Some(&refusing.url), 20);
assert!(refusing.received("job-t") > 0);
assert!(owed.iter().any(|e| e.request_id == "job-t"), "{owed:?}");
let hub = BookHub::start();
let (_, owed) = restart(&dir, Some(&hub.url), 1);
assert!(owed.is_empty(), "{owed:?}");
assert_eq!(hub.received("job-t"), 1);
assert_eq!(hub.booked("job-t", "job_execution"), Some(4.0));
let _ = std::fs::remove_dir_all(&dir);
}
}
#[test]
fn a_422_chunk_still_delivers_its_innocent_rows_and_keeps_the_bad_one_owed() {
let dir = wal_dir("invalid");
write_wal(
&dir,
&[
executed("job-ok-1"),
executed("job-bad"),
executed("job-ok-2"),
],
);
let hub = BookHub::with(Mode::InvalidRow("job-bad"));
let (_, owed) = restart(&dir, Some(&hub.url), 20);
let owed: Vec<_> = owed.iter().map(|e| e.request_id.as_str()).collect();
assert_eq!(owed, ["job-bad"]);
assert_eq!(hub.booked("job-ok-1", "job_execution"), Some(4.0));
assert_eq!(hub.booked("job-ok-2", "job_execution"), Some(4.0));
assert_eq!(hub.booked("job-bad", "job_execution"), None);
let fixed = BookHub::start();
let (_, owed) = restart(&dir, Some(&fixed.url), 1);
assert!(owed.is_empty(), "{owed:?}");
assert_eq!(fixed.booked("job-bad", "job_execution"), Some(4.0));
assert_eq!(
fixed.received("job-ok-1"),
0,
"delivered rows are not re-sent"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn bisecting_a_refused_chunk_is_bounded_per_flush() {
let dir = wal_dir("bisect");
let entries: Vec<_> = (0..300)
.map(|i| executed(&format!("job-{i}{}", if i % 10 == 0 { "-bad" } else { "" })))
.collect();
write_wal(&dir, &entries);
let hub = BookHub::with(Mode::InvalidSuffix("-bad"));
let wal = Wal::open(dir.join("wal.jsonl")).unwrap();
let ledger = Ledger::new(Some(hub.url.clone()), Some("k".to_string()));
let tx_buffer = TransactionBuffer::new();
replay_wal(&wal, &ledger, &tx_buffer, None);
for _ in 0..200 {
let before = hub.requests();
tx_buffer.flush_to_api(&hub.url, "k", &wal);
let spent = hub.requests() - before;
assert!(spent <= 1 + 64, "one flush sent {spent} requests");
if tx_buffer.pending_count() == 0 {
break;
}
}
assert_eq!(tx_buffer.pending_count(), 0, "the backlog drained");
for i in (0..300).filter(|i| i % 10 != 0) {
assert_eq!(
hub.booked(&format!("job-{i}"), "job_execution"),
Some(4.0),
"job-{i}"
);
}
wal.flush_buffer().unwrap();
let owed = wal.read_uncommitted().unwrap();
assert_eq!(owed.len(), 30, "only the malformed rows stay owed");
assert!(owed.iter().all(|e| e.request_id.ends_with("-bad")));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_debit_quarantined_while_the_hub_is_down_is_delivered_after_it() {
let dir = wal_dir("outage");
write_wal(&dir, &[executed("job-o")]);
let (_, owed) = restart(&dir, Some(DOWN), 20);
assert!(owed.iter().any(|e| e.request_id == "job-o"), "{owed:?}");
let hub = BookHub::start();
let (_, owed) = restart(&dir, Some(&hub.url), 1);
assert!(owed.is_empty(), "{owed:?}");
assert_eq!(hub.booked("job-o", "job_execution"), Some(4.0));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_hubless_broker_closes_executed_entries_and_mints_nothing() {
let dir = wal_dir("hubless");
let hubless = |rid: &str| WalEntry {
hubless: true,
receipt_hash: None,
..executed(rid)
};
write_wal(
&dir,
&[hubless("job-1"), hubless("job-2"), hubless("job-3")],
);
for _ in 0..2 {
let (ledger, owed) = restart(&dir, None, 0);
assert!(owed.is_empty(), "{owed:?}");
assert!(
local_balance(&ledger, "u1").abs() < 1e-9,
"no refund minted"
);
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_hubless_broker_closes_earned_entries_and_mints_nothing() {
let dir = wal_dir("hubless-earned");
let earned = |rid: &str| WalEntry {
request_id: format!("earn-{rid}"),
user_id: "owner".to_string(),
reservation_id: String::new(),
actual_cost: Some(3.6),
estimated_cost: 3.6,
status: WalStatus::Earned,
hubless: true,
receipt_hash: None,
..executed(rid)
};
write_wal(&dir, &[earned("job-1"), earned("job-2")]);
for _ in 0..2 {
let (ledger, owed) = restart(&dir, None, 0);
assert!(owed.is_empty(), "{owed:?}");
assert!(
local_balance(&ledger, "owner").abs() < 1e-9,
"no earn minted"
);
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_hub_brokers_entries_survive_a_restart_without_its_hub() {
let dir = wal_dir("missing-secret");
let earned = WalEntry {
request_id: "earn-job-1".to_string(),
user_id: "owner".to_string(),
reservation_id: String::new(),
actual_cost: Some(3.6),
estimated_cost: 3.6,
status: WalStatus::Earned,
..executed("job-1")
};
write_wal(&dir, &[executed("job-1"), earned]);
for _ in 0..2 {
let (ledger, owed) = restart(&dir, None, 0);
assert_eq!(owed.len(), 2, "both still owed: {owed:?}");
assert!(local_balance(&ledger, "u1").abs() < 1e-9);
assert!(local_balance(&ledger, "owner").abs() < 1e-9);
}
let hub = BookHub::start();
let (_, owed) = restart(&dir, Some(&hub.url), 1);
assert!(owed.is_empty(), "{owed:?}");
assert_eq!(hub.booked("job-1", "job_execution"), Some(4.0));
assert_eq!(hub.booked("earn-job-1", "earning"), Some(3.6));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_broker_with_a_hub_keeps_earned_entries_owed_until_delivered() {
let dir = wal_dir("hub-earned");
let earned = WalEntry {
request_id: "earn-job-1".to_string(),
user_id: "owner".to_string(),
reservation_id: String::new(),
actual_cost: Some(3.6),
estimated_cost: 3.6,
status: WalStatus::Earned,
..executed("job-1")
};
write_wal(&dir, &[earned]);
let (_, owed) = restart(&dir, Some(DOWN), 1);
assert!(
owed.iter().any(|e| e.request_id == "earn-job-1"),
"{owed:?}"
);
let hub = BookHub::start();
let (_, owed) = restart(&dir, Some(&hub.url), 1);
assert!(owed.is_empty(), "{owed:?}");
assert_eq!(hub.booked("earn-job-1", "earning"), Some(3.6));
let _ = std::fs::remove_dir_all(&dir);
}
}
#[cfg(all(test, unix))]
mod kill9_tests {
use super::test_hub::BookHub;
use super::*;
use crate::broker::wal::{read_all_from_disk, PayoutTo, WalEntry, WalPayout};
use std::io::{BufRead, BufReader, Write};
use std::os::unix::process::ExitStatusExt;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::time::{Duration, Instant};
const ENV_DIR: &str = "ZC_KILL9_WAL_DIR";
const ENV_BOUNDARY: &str = "ZC_KILL9_BOUNDARY";
const READY: &str = "ZC_KILL9_READY";
const RID: &str = "kill9-job";
const HELD: f64 = 10.0;
const COST: f64 = 4.0;
const C: f64 = 0.1;
fn entry(request_id: &str, user: &str, status: WalStatus, amount: f64) -> WalEntry {
WalEntry {
request_id: request_id.to_string(),
user_id: user.to_string(),
reservation_id: format!("res-{RID}"),
estimated_cost: amount,
actual_cost: (status != WalStatus::Reserved).then_some(amount),
worker_id: "wid".to_string(),
worker_name: Some("own-worker".to_string()),
duration_ms: Some(1000.0),
timestamp: chrono::Utc::now(),
status,
voucher_nonce: None,
unit: crate::broker::wal::WAL_UNIT.to_string(),
payout: None,
hubless: false,
receipt_hash: None,
}
}
#[test]
fn kill9_child() {
let (Ok(dir), Ok(boundary)) = (std::env::var(ENV_DIR), std::env::var(ENV_BOUNDARY)) else {
return;
};
let path = Path::new(&dir).join("wal.jsonl");
let wal = Arc::new(Wal::open(&path).unwrap());
let running = Arc::new(AtomicBool::new(true));
let (w, r) = (wal.clone(), running.clone());
std::thread::spawn(move || w.run_flush_worker(&r));
wal.append(&entry(RID, "u1", WalStatus::Reserved, HELD))
.unwrap();
let mut lines = 1;
if boundary != "reserved" {
wal.mark_executed(
RID,
COST,
1000.0,
Some(WalPayout {
to: PayoutTo::Owner {
user_id: "owner".to_string(),
},
row_id: format!("earn-{RID}"),
amount: COST * (1.0 - C),
commission: COST * C,
commission_c: C,
}),
None,
)
.unwrap();
lines += 1;
}
if boundary == "earned" {
let earn = format!("earn-{RID}");
wal.append(&entry(&earn, "owner", WalStatus::Earned, COST * (1.0 - C)))
.unwrap();
lines += 1;
}
let deadline = Instant::now() + Duration::from_secs(10);
while read_all_from_disk(&path).map_or(0, |e| e.len()) < lines {
assert!(Instant::now() < deadline, "flush worker never persisted");
std::thread::sleep(Duration::from_millis(5));
}
println!("{READY}");
std::io::stdout().flush().unwrap();
std::thread::sleep(Duration::from_secs(60));
std::process::exit(3);
}
struct Reaper(Child);
impl Drop for Reaper {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
fn sigkill_at(boundary: &str) -> PathBuf {
let dir =
std::env::temp_dir().join(format!("zc-kill9-{boundary}-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let module = module_path!().split_once("::").unwrap().1;
let child = Command::new(std::env::current_exe().unwrap())
.args([
&format!("{module}::kill9_child"),
"--exact",
"--nocapture",
"--test-threads=1",
])
.env(ENV_DIR, &dir)
.env(ENV_BOUNDARY, boundary)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.spawn()
.unwrap();
let mut child = Reaper(child);
let stdout = child.0.stdout.take().unwrap();
let (ready_tx, ready_rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
for line in BufReader::new(stdout).lines().map_while(Result::ok) {
if line.trim_end().ends_with(READY) {
let _ = ready_tx.send(());
}
}
});
ready_rx
.recv_timeout(Duration::from_secs(60))
.expect("the child never reached its boundary");
child.0.kill().unwrap(); let status = child.0.wait().unwrap();
assert_eq!(
status.signal(),
Some(9),
"child was not SIGKILLed: {status:?}"
);
dir
}
fn restart_twice(dir: &Path, hub: &BookHub, hub_up_at_replay: bool) {
let at_replay = if hub_up_at_replay {
hub.url.clone()
} else {
"http://127.0.0.1:1".to_string() };
for _ in 0..2 {
let wal = Wal::open(dir.join("wal.jsonl")).unwrap();
let ledger = Ledger::new(Some(at_replay.clone()), Some("k".to_string()));
let tx_buffer = TransactionBuffer::new();
replay_wal(&wal, &ledger, &tx_buffer, None);
tx_buffer.flush_to_api(&hub.url, "k", &wal);
wal.flush_buffer().unwrap();
}
for (rid, n) in hub.received.lock().unwrap().iter() {
assert_eq!(*n, 1, "{rid} delivered {n} times");
}
}
fn settle(boundary: &str, hub_up_at_replay: bool) -> (Option<f64>, Option<f64>) {
let dir = sigkill_at(boundary);
let hub = BookHub::start();
restart_twice(&dir, &hub, hub_up_at_replay);
let _ = std::fs::remove_dir_all(&dir);
(
hub.booked(RID, "job_execution"),
hub.booked(&format!("earn-{RID}"), "earning"),
)
}
fn assert_conserves(debit: Option<f64>, earn: Option<f64>) {
let (Some(debit), Some(earn)) = (debit, earn) else {
panic!("requester debit {debit:?}, owner earn {earn:?}: one leg was dropped");
};
assert!((debit - COST).abs() < 1e-9, "debit {debit} != {COST}");
assert!(
(debit - (earn + COST * C)).abs() < 1e-9,
"debit {debit} != earn {earn} + commission {}",
COST * C
);
}
#[test]
fn kill9_after_reserved_settles_nothing() {
for hub_up in [true, false] {
assert_eq!(settle("reserved", hub_up), (None, None), "hub up: {hub_up}");
}
}
#[test]
fn kill9_after_earned_redelivers_debit_and_earn() {
let (debit, earn) = settle("earned", true);
assert_conserves(debit, earn);
}
#[test]
fn kill9_after_earned_with_the_hub_down_at_restart_redelivers_debit_and_earn() {
let (debit, earn) = settle("earned", false);
assert_conserves(debit, earn);
}
#[test]
fn kill9_after_executed_redelivers_the_debit() {
for hub_up in [true, false] {
let (debit, _) = settle("executed", hub_up);
assert_eq!(debit, Some(COST), "hub up: {hub_up}");
}
}
#[test]
fn kill9_after_executed_redelivers_debit_and_earn() {
for hub_up in [true, false] {
let (debit, earn) = settle("executed", hub_up);
assert_conserves(debit, earn);
}
}
}