1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
//! WAL replay on broker startup.
//!
//! Reads uncommitted WAL entries and:
//! - Reserved entries → cancel (refund in-memory, queue for API flush)
//! - Executed entries → commit (refund diff in-memory) + publish transaction via API
use super::flush::{BufferedTransaction, TransactionBuffer};
use super::ledger::Ledger;
use super::wal::{Wal, WalStatus};
/// Replay uncommitted WAL entries on startup.
///
/// `tx_buffer` and `node_name` are used to re-deliver `Earned` entries (peer
/// earns that were booked in-memory but never confirmed flushed to the
/// dashboard) via the normal flush path — see the `WalStatus::Earned` arm
/// below.
pub fn replay_wal(
wal: &Wal,
ledger: &Ledger,
tx_buffer: &TransactionBuffer,
node_name: Option<&str>,
) {
let uncommitted = match wal.read_uncommitted() {
Ok(entries) => entries,
Err(e) => {
eprintln!(" [WAL] Failed to read uncommitted entries: {}", e);
return;
}
};
if uncommitted.is_empty() {
return;
}
println!(
" [WAL] Replaying {} uncommitted entries...",
uncommitted.len()
);
for entry in &uncommitted {
match entry.status {
WalStatus::Reserved => {
// Worker never returned — refund the reserved amount in-memory
println!(
" [WAL] Cancelling stale reservation for user {} (req: {}, cost: {:.6})",
entry.user_id, entry.request_id, entry.estimated_cost
);
if let Err(e) = ledger.cancel_from_wal(&entry.user_id, entry.estimated_cost) {
eprintln!(" [WAL] Failed to cancel for {}: {}", entry.request_id, e);
}
let _ = wal.update_status(&entry.request_id, WalStatus::Failed, None, None);
}
WalStatus::Executed => {
// Worker completed but commit didn't happen — commit now
let actual_cost = entry.actual_cost.unwrap_or(entry.estimated_cost);
println!(
" [WAL] Recovering executed request {} for user {} (cost: {:.6})",
entry.request_id, entry.user_id, actual_cost
);
// Publish transaction via dashboard API. The hub knows this
// worker by its stable registered name (`sync_workers_via_api`
// registers workers keyed on `Worker.name`), never by the
// ephemeral `Worker.id` minted per registration — publish
// `worker_name` when the WAL recorded one (zc#272). Older WAL
// lines predate that field and deserialize it as `None`; they
// fall back to `worker_id`, preserving today's behavior for
// in-flight entries written before this fix shipped.
let worker_identifier = entry.worker_name.as_deref().unwrap_or(&entry.worker_id);
ledger.publish_transaction_for_voucher(
&entry.request_id,
&entry.user_id,
"commit",
actual_cost,
0.0,
crate::broker::worker::WorkerName::reported(worker_identifier),
entry.duration_ms.unwrap_or(0.0),
None,
entry.voucher_nonce.as_deref(),
);
// Refund difference (reserved - actual) in-memory
match ledger.commit_from_wal(&entry.user_id, entry.estimated_cost, actual_cost) {
Ok(balance) => {
println!(
" [WAL] Recovered: user {} balance now {:.4}",
entry.user_id, balance
);
}
Err(e) => {
eprintln!(" [WAL] Failed to commit for {}: {}", entry.request_id, e);
}
}
let _ = wal.update_status(
&entry.request_id,
WalStatus::Committed,
entry.actual_cost,
entry.duration_ms,
);
}
WalStatus::Earned => {
// Worker earn was booked in-memory but never confirmed
// flushed to the dashboard (crash before/while flushing).
//
// We must NOT re-apply the in-memory earning here. The dashboard
// is authoritative for spendable balance: `authoritative_balances`
// re-seeds lazily from the dashboard (via load_balance_if_needed)
// the next time this user is touched, and on the crash-after-
// dashboard-write window the dashboard ALREADY reflects this
// earn. Calling local_add would seed-from-dashboard and
// then add `amount` on top, inflating the in-memory spendable
// balance by `amount` until a clean restart (bounded over-spend).
//
// So replay ONLY re-delivers the earn to the dashboard to make it
// durable there: re-buffer the same `earn-{request_id}` earning tx
// (idempotent via dashboard request_id dedup) and leave the WAL
// entry `Earned`, so flush_to_api marks it Committed once the
// dashboard confirms it applied/duplicate. balance_after is only
// metadata on the tx row; use the ledger's current view without
// mutating it.
let amount = entry.actual_cost.unwrap_or(entry.estimated_cost);
println!(
" [WAL] Re-delivering unflushed earn {} for user {} (amount: {:.6})",
entry.request_id, entry.user_id, amount
);
let balance_after = ledger.get_balance(&entry.user_id);
tx_buffer.push_transaction(BufferedTransaction {
request_id: entry.request_id.clone(),
user_id: entry.user_id.clone(),
tx_type: "earning".to_string(),
amount,
balance_after,
worker_id: entry.worker_id.clone(),
duration_ms: entry.duration_ms.unwrap_or(0.0),
source_node: node_name.map(|s| s.to_string()),
worker_name: entry
.worker_name
.clone()
.or_else(|| Some(entry.worker_id.clone())),
worker_uri: None,
price_per_hour: 0.0,
voucher_nonce: None,
});
}
// Committed/Failed are filtered out by read_uncommitted
_ => {}
}
}
// Compact after replay
if let Err(e) = wal.compact() {
eprintln!(" [WAL] Failed to compact after replay: {}", e);
}
println!(" [WAL] Replay complete");
}
#[cfg(test)]
mod worker_attribution_tests {
use super::*;
use crate::broker::wal::WalEntry;
use std::sync::{Arc, Mutex};
fn tmp_wal(name: &str) -> std::path::PathBuf {
std::path::PathBuf::from(format!("/tmp/zc_recovery_test_{}.jsonl", name))
}
fn cleanup(p: &std::path::Path) {
let _ = std::fs::remove_file(p);
let _ = std::fs::remove_file(p.with_extension("tmp"));
}
/// Mock hub on a fresh loopback port. Captures the raw body of the
/// `/api/broker/transaction` POST and answers *every* request 200 until
/// it has been idle for a few seconds.
///
/// It must keep serving: `replay_wal` makes more than one hub call (the
/// publish POST, then a balance GET via `ledger.get_balance`). The old
/// one-shot server answered the POST and exited, so the follow-up GET
/// could land on a dead listener or a pooled keep-alive connection nobody
/// reads from, and `ureq` (no default timeout) blocked forever. That was
/// the intermittent >420s hang in CI.
fn capturing_hub() -> (String, Arc<Mutex<Option<String>>>) {
let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
let port = server.server_addr().to_ip().unwrap().port();
let captured = Arc::new(Mutex::new(None));
let seen = captured.clone();
std::thread::spawn(move || {
while let Ok(Some(mut req)) = server.recv_timeout(std::time::Duration::from_secs(5)) {
let mut body = String::new();
let _ = req.as_reader().read_to_string(&mut body);
if req.url().ends_with("/api/broker/transaction") {
*seen.lock().unwrap() = Some(body);
}
let _ = req.respond(tiny_http::Response::from_string("{}").with_status_code(200));
}
});
(format!("http://127.0.0.1:{port}"), captured)
}
/// Run `replay_wal` on a helper thread and fail fast if it does not
/// return, instead of hanging CI until the job timeout.
fn replay_bounded(wal: Wal, ledger: Ledger, tx_buffer: TransactionBuffer) {
let (done_tx, done_rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
replay_wal(&wal, &ledger, &tx_buffer, None);
let _ = done_tx.send(());
});
done_rx
.recv_timeout(std::time::Duration::from_secs(30))
.expect("replay_wal did not return within 30s (blocked on a hub call?)");
}
fn wait_for_json(captured: &Arc<Mutex<Option<String>>>) -> serde_json::Value {
for _ in 0..300 {
if let Some(body) = captured.lock().unwrap().clone() {
return serde_json::from_str(&body)
.unwrap_or_else(|e| panic!("captured body {body:?} was not JSON: {e}"));
}
std::thread::sleep(std::time::Duration::from_millis(10));
}
panic!("the mock hub never received the dashboard POST");
}
fn executed_entry(request_id: &str, worker_id: &str, worker_name: Option<&str>) -> WalEntry {
WalEntry {
request_id: request_id.to_string(),
user_id: "alice".to_string(),
reservation_id: format!("res-{request_id}"),
estimated_cost: 0.05,
actual_cost: Some(0.05),
worker_id: worker_id.to_string(),
worker_name: worker_name.map(str::to_string),
duration_ms: Some(1000.0),
timestamp: chrono::Utc::now(),
status: WalStatus::Executed,
voucher_nonce: None,
unit: crate::broker::wal::WAL_UNIT.to_string(),
}
}
/// The Executed arm publishes straight to the dashboard via
/// `ledger.publish_transaction`. It must send the WAL-recorded
/// `worker_name` (the identifier the hub actually registered the worker
/// under, per `sync_workers_via_api`), not the ephemeral `worker_id` the
/// original `Reserved` entry was appended with (zc#272).
#[test]
fn replay_of_an_executed_entry_publishes_the_workers_recorded_name() {
let path = tmp_wal("executed_with_name");
cleanup(&path);
let wal = Wal::open(&path).unwrap();
wal.append(&executed_entry(
"req-exec-1",
"ephemeral-uuid-1234",
Some("gpu-box-1"),
))
.unwrap();
wal.flush_buffer().unwrap();
let (hub, captured) = capturing_hub();
let ledger = Ledger::new(Some(hub), Some("zk_1_test".to_string()));
let tx_buffer = TransactionBuffer::new();
replay_bounded(wal, ledger, tx_buffer);
let body = wait_for_json(&captured);
assert_eq!(
body["worker_id"], "gpu-box-1",
"replay must publish the worker's registered name, body: {body}"
);
assert_ne!(
body["worker_id"], "ephemeral-uuid-1234",
"must not leak the ephemeral registry id to the hub, body: {body}"
);
assert_eq!(body["request_id"], "req-exec-1", "body: {body}");
cleanup(&path);
}
/// Old WAL lines written before `worker_name` existed deserialize with
/// `None` (`#[serde(default)]`) — replay must still fall back to
/// `worker_id` rather than sending an empty/null identifier.
#[test]
fn replay_of_an_executed_entry_without_a_recorded_name_falls_back_to_worker_id() {
let path = tmp_wal("executed_without_name");
cleanup(&path);
let wal = Wal::open(&path).unwrap();
wal.append(&executed_entry("req-exec-2", "legacy-worker-id", None))
.unwrap();
wal.flush_buffer().unwrap();
let (hub, captured) = capturing_hub();
let ledger = Ledger::new(Some(hub), Some("zk_1_test".to_string()));
let tx_buffer = TransactionBuffer::new();
replay_bounded(wal, ledger, tx_buffer);
let body = wait_for_json(&captured);
assert_eq!(body["worker_id"], "legacy-worker-id", "body: {body}");
cleanup(&path);
}
/// The Earned arm re-buffers the earning for the normal flush path
/// (`flush::flush_to_api`), which sends `tx.worker_name` to the hub. It
/// must carry forward the WAL-recorded name, not the ephemeral id.
#[test]
fn replay_of_an_earned_entry_rebuffers_the_workers_recorded_name() {
let path = tmp_wal("earned_with_name");
cleanup(&path);
let wal = Wal::open(&path).unwrap();
wal.append(&WalEntry {
request_id: "earn-req-1".to_string(),
user_id: "owner".to_string(),
reservation_id: String::new(),
estimated_cost: 1.0,
actual_cost: Some(1.0),
worker_id: "ephemeral-uuid-5678".to_string(),
worker_name: Some("gpu-box-2".to_string()),
duration_ms: Some(500.0),
timestamp: chrono::Utc::now(),
status: WalStatus::Earned,
voucher_nonce: None,
unit: crate::broker::wal::WAL_UNIT.to_string(),
})
.unwrap();
wal.flush_buffer().unwrap();
let ledger = Ledger::new(None, None);
let tx_buffer = TransactionBuffer::new();
replay_wal(&wal, &ledger, &tx_buffer, None);
let queued = tx_buffer.pending_snapshot();
let tx = queued
.iter()
.find(|t| t.request_id == "earn-req-1")
.expect("the unflushed earn must be re-buffered");
assert_eq!(
tx.worker_name,
Some("gpu-box-2".to_string()),
"re-buffered earn must carry the worker's recorded name, not its ephemeral id"
);
cleanup(&path);
}
}