zc2 0.0.42

P2P compute broker with credit-based billing, WAL, and broker mesh support
//! WAL replay on broker startup.
//!
//! Reads uncommitted WAL entries and:
//! - Reserved entries → cancel (refund credits in-memory, queue for API flush)
//! - Executed entries → commit (refund diff in-memory) + publish transaction via API

use super::flush::{BufferedTransaction, TransactionBuffer};
use super::ledger::Ledger;
use super::wal::{Wal, WalStatus};

/// Replay uncommitted WAL entries on startup.
///
/// `tx_buffer` and `node_name` are used to re-deliver `Earned` entries (peer
/// earns that were credited in-memory but never confirmed flushed to the
/// dashboard) via the normal flush path — see the `WalStatus::Earned` arm
/// below.
pub fn replay_wal(
    wal: &Wal,
    ledger: &Ledger,
    tx_buffer: &TransactionBuffer,
    node_name: Option<&str>,
) {
    let uncommitted = match wal.read_uncommitted() {
        Ok(entries) => entries,
        Err(e) => {
            eprintln!("  [WAL] Failed to read uncommitted entries: {}", e);
            return;
        }
    };

    if uncommitted.is_empty() {
        return;
    }

    println!(
        "  [WAL] Replaying {} uncommitted entries...",
        uncommitted.len()
    );

    for entry in &uncommitted {
        match entry.status {
            WalStatus::Reserved => {
                // Worker never returned — refund the reserved amount in-memory
                println!(
                    "  [WAL] Cancelling stale reservation for user {} (req: {}, cost: {:.6})",
                    entry.user_id, entry.request_id, entry.estimated_cost
                );
                if let Err(e) = ledger.cancel_from_wal(&entry.user_id, entry.estimated_cost) {
                    eprintln!("  [WAL] Failed to cancel for {}: {}", entry.request_id, e);
                }
                let _ = wal.update_status(&entry.request_id, WalStatus::Failed, None, None);
            }
            WalStatus::Executed => {
                // Worker completed but commit didn't happen — commit now
                let actual_cost = entry.actual_cost.unwrap_or(entry.estimated_cost);
                println!(
                    "  [WAL] Recovering executed request {} for user {} (cost: {:.6})",
                    entry.request_id, entry.user_id, actual_cost
                );

                // Publish transaction via dashboard API. The hub knows this
                // worker by its stable registered name (`sync_workers_via_api`
                // registers workers keyed on `Worker.name`), never by the
                // ephemeral `Worker.id` minted per registration — publish
                // `worker_name` when the WAL recorded one (zc#272). Older WAL
                // lines predate that field and deserialize it as `None`; they
                // fall back to `worker_id`, preserving today's behavior for
                // in-flight entries written before this fix shipped.
                let worker_identifier = entry.worker_name.as_deref().unwrap_or(&entry.worker_id);
                ledger.publish_transaction_for_voucher(
                    &entry.request_id,
                    &entry.user_id,
                    "commit",
                    actual_cost,
                    0.0,
                    crate::broker::worker::WorkerName::reported(worker_identifier),
                    entry.duration_ms.unwrap_or(0.0),
                    None,
                    entry.voucher_nonce.as_deref(),
                );

                // Refund difference (reserved - actual) in-memory
                match ledger.commit_from_wal(&entry.user_id, entry.estimated_cost, actual_cost) {
                    Ok(balance) => {
                        println!(
                            "  [WAL] Recovered: user {} balance now {:.4}",
                            entry.user_id, balance
                        );
                    }
                    Err(e) => {
                        eprintln!("  [WAL] Failed to commit for {}: {}", entry.request_id, e);
                    }
                }

                let _ = wal.update_status(
                    &entry.request_id,
                    WalStatus::Committed,
                    entry.actual_cost,
                    entry.duration_ms,
                );
            }
            WalStatus::Earned => {
                // Worker earn was credited in-memory but never confirmed
                // flushed to the dashboard (crash before/while flushing).
                //
                // We must NOT re-apply the in-memory credit here. The dashboard
                // is authoritative for spendable balance: `authoritative_balances`
                // re-seeds lazily from the dashboard (via load_balance_if_needed)
                // the next time this user is touched, and on the crash-after-
                // dashboard-write window the dashboard ALREADY reflects this
                // earn. Calling local_add_credits would seed-from-dashboard and
                // then add `amount` on top, inflating the in-memory spendable
                // balance by `amount` until a clean restart (bounded over-spend).
                //
                // So replay ONLY re-delivers the earn to the dashboard to make it
                // durable there: re-buffer the same `earn-{request_id}` credit tx
                // (idempotent via dashboard request_id dedup) and leave the WAL
                // entry `Earned`, so flush_to_api marks it Committed once the
                // dashboard confirms it applied/duplicate. balance_after is only
                // metadata on the tx row; use the ledger's current view without
                // mutating it.
                let amount = entry.actual_cost.unwrap_or(entry.estimated_cost);
                println!(
                    "  [WAL] Re-delivering unflushed earn {} for user {} (amount: {:.6})",
                    entry.request_id, entry.user_id, amount
                );
                let balance_after = ledger.get_balance(&entry.user_id);
                tx_buffer.push_transaction(BufferedTransaction {
                    request_id: entry.request_id.clone(),
                    user_id: entry.user_id.clone(),
                    tx_type: "credit".to_string(),
                    amount,
                    balance_after,
                    worker_id: entry.worker_id.clone(),
                    duration_ms: entry.duration_ms.unwrap_or(0.0),
                    source_node: node_name.map(|s| s.to_string()),
                    worker_name: entry
                        .worker_name
                        .clone()
                        .or_else(|| Some(entry.worker_id.clone())),
                    worker_uri: None,
                    price_per_hour: 0.0,
                    voucher_nonce: None,
                });
            }
            // Committed/Failed are filtered out by read_uncommitted
            _ => {}
        }
    }

    // Compact after replay
    if let Err(e) = wal.compact() {
        eprintln!("  [WAL] Failed to compact after replay: {}", e);
    }

    println!("  [WAL] Replay complete");
}

#[cfg(test)]
mod worker_attribution_tests {
    use super::*;
    use crate::broker::wal::WalEntry;
    use std::sync::{Arc, Mutex};

    fn tmp_wal(name: &str) -> std::path::PathBuf {
        std::path::PathBuf::from(format!("/tmp/zc_recovery_test_{}.jsonl", name))
    }

    fn cleanup(p: &std::path::Path) {
        let _ = std::fs::remove_file(p);
        let _ = std::fs::remove_file(p.with_extension("tmp"));
    }

    /// Captures the raw request body of the first request a fresh loopback
    /// listener receives, and answers it 200. Mirrors the mock hub in
    /// `ledger::publish_transaction_wire_format_tests`.
    fn capturing_hub() -> (String, Arc<Mutex<Option<String>>>) {
        let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
        let port = server.server_addr().to_ip().unwrap().port();
        let captured = Arc::new(Mutex::new(None));
        let seen = captured.clone();
        std::thread::spawn(move || {
            if let Ok(mut req) = server.recv() {
                let mut body = String::new();
                let _ = req.as_reader().read_to_string(&mut body);
                *seen.lock().unwrap() = Some(body);
                let _ = req.respond(tiny_http::Response::from_string("{}").with_status_code(200));
            }
        });
        (format!("http://127.0.0.1:{port}"), captured)
    }

    fn wait_for_json(captured: &Arc<Mutex<Option<String>>>) -> serde_json::Value {
        for _ in 0..300 {
            if let Some(body) = captured.lock().unwrap().clone() {
                return serde_json::from_str(&body)
                    .unwrap_or_else(|e| panic!("captured body {body:?} was not JSON: {e}"));
            }
            std::thread::sleep(std::time::Duration::from_millis(10));
        }
        panic!("the mock hub never received the dashboard POST");
    }

    fn executed_entry(request_id: &str, worker_id: &str, worker_name: Option<&str>) -> WalEntry {
        WalEntry {
            request_id: request_id.to_string(),
            user_id: "alice".to_string(),
            reservation_id: format!("res-{request_id}"),
            estimated_cost: 0.05,
            actual_cost: Some(0.05),
            worker_id: worker_id.to_string(),
            worker_name: worker_name.map(str::to_string),
            duration_ms: Some(1000.0),
            timestamp: chrono::Utc::now(),
            status: WalStatus::Executed,
            voucher_nonce: None,
        }
    }

    /// The Executed arm publishes straight to the dashboard via
    /// `ledger.publish_transaction`. It must send the WAL-recorded
    /// `worker_name` (the identifier the hub actually registered the worker
    /// under, per `sync_workers_via_api`), not the ephemeral `worker_id` the
    /// original `Reserved` entry was appended with (zc#272).
    #[test]
    fn replay_of_an_executed_entry_publishes_the_workers_recorded_name() {
        let path = tmp_wal("executed_with_name");
        cleanup(&path);
        let wal = Wal::open(&path).unwrap();
        wal.append(&executed_entry(
            "req-exec-1",
            "ephemeral-uuid-1234",
            Some("gpu-box-1"),
        ))
        .unwrap();
        wal.flush_buffer().unwrap();

        let (hub, captured) = capturing_hub();
        let ledger = Ledger::new(Some(hub), Some("zk_1_test".to_string()));
        let tx_buffer = TransactionBuffer::new();

        replay_wal(&wal, &ledger, &tx_buffer, None);

        let body = wait_for_json(&captured);
        assert_eq!(
            body["worker_id"], "gpu-box-1",
            "replay must publish the worker's registered name, body: {body}"
        );
        assert_ne!(
            body["worker_id"], "ephemeral-uuid-1234",
            "must not leak the ephemeral registry id to the hub, body: {body}"
        );
        assert_eq!(body["request_id"], "req-exec-1", "body: {body}");

        cleanup(&path);
    }

    /// Old WAL lines written before `worker_name` existed deserialize with
    /// `None` (`#[serde(default)]`) — replay must still fall back to
    /// `worker_id` rather than sending an empty/null identifier.
    #[test]
    fn replay_of_an_executed_entry_without_a_recorded_name_falls_back_to_worker_id() {
        let path = tmp_wal("executed_without_name");
        cleanup(&path);
        let wal = Wal::open(&path).unwrap();
        wal.append(&executed_entry("req-exec-2", "legacy-worker-id", None))
            .unwrap();
        wal.flush_buffer().unwrap();

        let (hub, captured) = capturing_hub();
        let ledger = Ledger::new(Some(hub), Some("zk_1_test".to_string()));
        let tx_buffer = TransactionBuffer::new();

        replay_wal(&wal, &ledger, &tx_buffer, None);

        let body = wait_for_json(&captured);
        assert_eq!(body["worker_id"], "legacy-worker-id", "body: {body}");

        cleanup(&path);
    }

    /// The Earned arm re-buffers the credit for the normal flush path
    /// (`flush::flush_to_api`), which sends `tx.worker_name` to the hub. It
    /// must carry forward the WAL-recorded name, not the ephemeral id.
    #[test]
    fn replay_of_an_earned_entry_rebuffers_the_workers_recorded_name() {
        let path = tmp_wal("earned_with_name");
        cleanup(&path);
        let wal = Wal::open(&path).unwrap();
        wal.append(&WalEntry {
            request_id: "earn-req-1".to_string(),
            user_id: "owner".to_string(),
            reservation_id: String::new(),
            estimated_cost: 1.0,
            actual_cost: Some(1.0),
            worker_id: "ephemeral-uuid-5678".to_string(),
            worker_name: Some("gpu-box-2".to_string()),
            duration_ms: Some(500.0),
            timestamp: chrono::Utc::now(),
            status: WalStatus::Earned,
            voucher_nonce: None,
        })
        .unwrap();
        wal.flush_buffer().unwrap();

        let ledger = Ledger::new(None, None);
        let tx_buffer = TransactionBuffer::new();

        replay_wal(&wal, &ledger, &tx_buffer, None);

        let queued = tx_buffer.pending_snapshot();
        let tx = queued
            .iter()
            .find(|t| t.request_id == "earn-req-1")
            .expect("the unflushed earn must be re-buffered");
        assert_eq!(
            tx.worker_name,
            Some("gpu-box-2".to_string()),
            "re-buffered earn must carry the worker's recorded name, not its ephemeral id"
        );

        cleanup(&path);
    }
}