zc2 0.0.50

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
//! Durable retry queue for voucher redeems (zc#571).
//!
//! A redeem that fails (hub down, 5xx, timeout) used to be logged and
//! forgotten: the hub never learned the nonce was spent, so the voucher
//! stayed redeemable and the platform commission was never booked. The
//! failed redeem is queued here, persisted in a small file beside the WAL so a
//! restart keeps it, and retried by the broker's periodic loop with backoff
//! until the hub answers or the voucher has been expired for
//! [`GIVE_UP_AFTER_EXPIRY_SECS`].

use std::path::PathBuf;
use std::sync::Mutex;

use serde::{Deserialize, Serialize};

use super::node_sync::RedeemOutcome;

/// Keep retrying this long after the voucher's own expiry.
pub const GIVE_UP_AFTER_EXPIRY_SECS: u64 = 3 * 24 * 3600;
/// First retry delay; doubles per attempt up to [`MAX_BACKOFF_SECS`].
const BASE_BACKOFF_SECS: u64 = 30;
const MAX_BACKOFF_SECS: u64 = 600;
/// Hard bound on queued redeems (oldest dropped first): the queue is a
/// safety net, not a log.
const MAX_QUEUED: usize = 10_000;
/// Redeems attempted per tick, so one tick never stalls the loop it runs in.
pub const MAX_PER_TICK: usize = 8;

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PendingRedeem {
    pub nonce: String,
    pub actual_cost: f64,
    /// Unix secs after which the redeem is abandoned.
    pub give_up_at: u64,
    #[serde(default)]
    pub next_try: u64,
    #[serde(default)]
    pub attempts: u32,
}

pub struct RedeemQueue {
    path: Option<PathBuf>,
    items: Mutex<Vec<PendingRedeem>>,
    /// Entries dropped because the hub refused the redeem for good (409).
    refused: std::sync::atomic::AtomicU64,
}

impl RedeemQueue {
    /// An in-memory queue (tests, brokers with no state dir).
    pub fn in_memory() -> Self {
        Self {
            path: None,
            items: Mutex::new(Vec::new()),
            refused: std::sync::atomic::AtomicU64::new(0),
        }
    }

    /// Open the queue persisted at `path`, loading what a previous run left.
    pub fn open(path: PathBuf) -> Self {
        let items: Vec<PendingRedeem> = std::fs::read_to_string(&path)
            .unwrap_or_default()
            .lines()
            .filter_map(|l| serde_json::from_str(l).ok())
            .collect();
        Self {
            path: Some(path),
            items: Mutex::new(items),
            refused: std::sync::atomic::AtomicU64::new(0),
        }
    }

    /// How many queued redeems the hub refused for good since start.
    pub fn refused(&self) -> u64 {
        self.refused.load(std::sync::atomic::Ordering::Relaxed)
    }

    pub fn depth(&self) -> usize {
        self.items.lock().unwrap_or_else(|e| e.into_inner()).len()
    }

    /// Queue a redeem that just failed. `voucher_exp` is the voucher's expiry
    /// (unix secs); a nonce already queued is left as it is.
    pub fn enqueue(&self, nonce: &str, actual_cost: f64, voucher_exp: u64, now: u64) {
        let mut g = self.items.lock().unwrap_or_else(|e| e.into_inner());
        if g.iter().any(|p| p.nonce == nonce) {
            return;
        }
        if g.len() >= MAX_QUEUED {
            g.remove(0);
        }
        g.push(PendingRedeem {
            nonce: nonce.to_string(),
            actual_cost,
            give_up_at: voucher_exp
                .saturating_add(GIVE_UP_AFTER_EXPIRY_SECS)
                .max(now + 60),
            next_try: now + BASE_BACKOFF_SECS,
            attempts: 0,
        });
        self.persist(&g);
    }

    /// Retry what is due. `redeem` is the hub call: `Ok(Redeemed)` or
    /// `Ok(AlreadyRedeemed)` settles the entry, `Ok(Refused)` (the hub never
    /// minted the nonce) drops it and counts it in [`refused`](Self::refused),
    /// `Err` backs it off. Entries past their give-up time are dropped.
    /// Returns how many entries were settled (a refusal is not one).
    pub fn tick_with(
        &self,
        now: u64,
        redeem: &dyn Fn(&str, f64) -> Result<RedeemOutcome, String>,
    ) -> usize {
        let due: Vec<PendingRedeem> = {
            let mut g = self.items.lock().unwrap_or_else(|e| e.into_inner());
            let before = g.len();
            g.retain(|p| p.give_up_at > now);
            if g.len() != before {
                self.persist(&g);
            }
            g.iter()
                .filter(|p| p.next_try <= now)
                .take(MAX_PER_TICK)
                .cloned()
                .collect()
        };
        let mut settled = 0;
        for p in due {
            // The hub call runs outside the lock.
            let outcome = redeem(&p.nonce, p.actual_cost);
            let mut g = self.items.lock().unwrap_or_else(|e| e.into_inner());
            match outcome {
                Ok(RedeemOutcome::Refused) => {
                    g.retain(|q| q.nonce != p.nonce);
                    self.refused
                        .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                }
                Ok(_) => {
                    g.retain(|q| q.nonce != p.nonce);
                    settled += 1;
                }
                Err(_) => {
                    if let Some(q) = g.iter_mut().find(|q| q.nonce == p.nonce) {
                        q.attempts += 1;
                        let back = (BASE_BACKOFF_SECS << q.attempts.min(5)).min(MAX_BACKOFF_SECS);
                        q.next_try = now + back;
                    }
                }
            }
            self.persist(&g);
        }
        settled
    }

    /// Rewrite the file (atomic rename). Best effort: the queue is a retry
    /// aid and the in-memory copy still serves this run.
    fn persist(&self, items: &[PendingRedeem]) {
        let Some(path) = &self.path else { return };
        let mut out = String::new();
        for p in items {
            if let Ok(l) = serde_json::to_string(p) {
                out.push_str(&l);
                out.push('\n');
            }
        }
        let tmp = path.with_extension("tmp");
        if std::fs::write(&tmp, out).is_ok() {
            let _ = std::fs::rename(&tmp, path);
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::Cell;

    #[test]
    fn a_failed_redeem_is_retried_until_it_succeeds() {
        let q = RedeemQueue::in_memory();
        q.enqueue("n1", 1.5, 10_000, 1_000);
        assert_eq!(q.depth(), 1);
        // not due yet
        assert_eq!(q.tick_with(1_000, &|_, _| Ok(RedeemOutcome::Redeemed)), 0);
        let calls = Cell::new(0);
        // hub down: stays queued, backs off
        let n = q.tick_with(1_031, &|_, _| {
            calls.set(calls.get() + 1);
            Err("down".into())
        });
        assert_eq!((n, calls.get(), q.depth()), (0, 1, 1));
        // still backing off
        assert_eq!(q.tick_with(1_032, &|_, _| Ok(RedeemOutcome::Redeemed)), 0);
        // hub back
        let seen = std::cell::RefCell::new(Vec::new());
        let n = q.tick_with(2_000, &|nonce, cost| {
            seen.borrow_mut().push((nonce.to_string(), cost));
            Ok(RedeemOutcome::Redeemed)
        });
        assert_eq!(n, 1);
        assert_eq!(q.depth(), 0);
        assert_eq!(seen.borrow()[0], ("n1".to_string(), 1.5));
    }

    #[test]
    fn already_spent_settles_the_entry() {
        let q = RedeemQueue::in_memory();
        q.enqueue("n1", 1.0, 10_000, 0);
        assert_eq!(
            q.tick_with(100, &|_, _| Ok(RedeemOutcome::AlreadyRedeemed)),
            1
        );
        assert_eq!(q.depth(), 0);
    }

    #[test]
    fn a_refused_redeem_is_dropped_and_counted_not_retried() {
        let q = RedeemQueue::in_memory();
        q.enqueue("ghost", 1.0, 10_000, 0);
        q.enqueue("real", 1.0, 10_000, 0);
        let n = q.tick_with(100, &|nonce, _| {
            Ok(if nonce == "ghost" {
                RedeemOutcome::Refused
            } else {
                RedeemOutcome::Redeemed
            })
        });
        assert_eq!((n, q.depth(), q.refused()), (1, 0, 1));
        // nothing left to retry
        assert_eq!(q.tick_with(10_000, &|_, _| Err("never".into())), 0);
    }

    #[test]
    fn a_long_expired_voucher_is_abandoned_and_a_nonce_is_queued_once() {
        let q = RedeemQueue::in_memory();
        q.enqueue("n1", 1.0, 100, 0);
        q.enqueue("n1", 9.0, 100, 0);
        assert_eq!(q.depth(), 1);
        let after = 100 + GIVE_UP_AFTER_EXPIRY_SECS + 1;
        assert_eq!(q.tick_with(after, &|_, _| Err("never called".into())), 0);
        assert_eq!(q.depth(), 0, "dropped, not retried");
    }

    #[test]
    fn the_queue_survives_a_restart() {
        let dir = std::env::temp_dir().join(format!("zc-vq-{}-{}", std::process::id(), line!()));
        std::fs::create_dir_all(&dir).unwrap();
        let path = dir.join("voucher-redeem-queue.jsonl");
        {
            let q = RedeemQueue::open(path.clone());
            q.enqueue("n1", 2.0, 10_000, 5);
        }
        let q = RedeemQueue::open(path.clone());
        assert_eq!(q.depth(), 1);
        assert_eq!(
            q.tick_with(1_000, &|n, c| {
                assert_eq!((n, c), ("n1", 2.0));
                Ok(RedeemOutcome::Redeemed)
            }),
            1
        );
        assert_eq!(
            RedeemQueue::open(path).depth(),
            0,
            "settled entry not reloaded"
        );
        let _ = std::fs::remove_dir_all(&dir);
    }
}