use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use alloy::dyn_abi::Eip712Domain;
use alloy::eips::BlockId;
use alloy::primitives::{Address, TxHash, U256};
use alloy::providers::Provider;
use alloy::signers::local::PrivateKeySigner;
use anyhow::{Context, Result};
use decdn_incentive::erc20::Erc20;
use decdn_incentive::payment_pool::{PaymentPool, enumerate_owned_pools_at};
use decdn_incentive::{
AdvanceOutcome, BuyerPoolState, BuyerPoolStore, Deployment, LaneKey, PoolId,
PoolOpenFailureReason,
};
use futures_util::FutureExt;
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, error, info, warn};
use crate::metrics::Metrics;
use decdn_client::buyer_pool::{
LOW_WATER_DIVISOR, OpenUnconfirmed, OpenedPool, PoolNotOpen, ProgressWrite, ToppedUpPool,
WatermarkRow, ensure_allowance, grade_deposit_credit, open_pool, opened_from_receipt,
pool_accepts_funds, refill_amount, self_owned_lane_ctx, send_open_pool, top_up as pool_top_up,
topped_up_effect,
};
use decdn_client::{LocalPullFault, PoolContext, PoolReplaced, Recovery};
use decdn_common::redact::{sanitize_err_chain, sanitize_error_sources};
const RECLAIM_SWEEP_INTERVAL: Duration = Duration::from_hours(1);
const WATERMARK_READ_TIMEOUT: Duration = Duration::from_secs(10);
const OPEN_RESOLVE_INTERVAL: Duration = Duration::from_secs(5);
const OPEN_RECONCILE_MAX_BACKOFF: Duration = Duration::from_mins(5);
const OPEN_UNRESOLVED_WARN_INTERVAL: Duration = Duration::from_mins(1);
const MAX_OPEN_TXS: usize = 4;
type SeedSlot = Arc<tokio::sync::Mutex<Option<(LaneKey, PaymentPool::Lane)>>>;
struct SeedTurn<'a> {
slots: &'a Mutex<HashMap<LaneKey, SeedSlot>>,
lane: LaneKey,
slot: SeedSlot,
}
impl Drop for SeedTurn<'_> {
fn drop(&mut self) {
let mut slots = self
.slots
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let last = Arc::strong_count(&self.slot) <= 2;
if last
&& slots
.get(&self.lane)
.is_some_and(|held| Arc::ptr_eq(held, &self.slot))
{
slots.remove(&self.lane);
}
}
}
#[derive(Debug)]
pub struct PoolOpenPending {
pub waited: Duration,
}
impl std::fmt::Display for PoolOpenPending {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"buyer pool open still in flight after {:?}; retry",
self.waited
)
}
}
impl std::error::Error for PoolOpenPending {}
#[derive(Debug)]
pub struct OpenReported;
impl std::fmt::Display for OpenReported {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "buyer pool open failure (already reported)")
}
}
impl std::error::Error for OpenReported {}
type OpenOutcome = Result<(), Arc<anyhow::Error>>;
type SharedOpen = futures_util::future::Shared<BoxFuture<'static, OpenOutcome>>;
type TopUpOutcome = Result<TopUpLanded, Arc<anyhow::Error>>;
type SharedTopUp = futures_util::future::Shared<BoxFuture<'static, TopUpOutcome>>;
type BoxFuture<'a, T> = std::pin::Pin<Box<dyn std::future::Future<Output = T> + Send + 'a>>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TopUpLanded {
pub new_deposit: U256,
pub added: U256,
}
#[derive(Debug)]
struct EscrowedUntracked;
impl std::fmt::Display for EscrowedUntracked {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "topUp escrowed but not credited to the local pool row")
}
}
impl std::error::Error for EscrowedUntracked {}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TopUpFunder {
Refill,
Recovery,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TopUpClaim {
Spawned,
Joined {
claimed: U256,
requested: U256,
},
}
struct InFlightTopUp {
fut: SharedTopUp,
requested: U256,
unclaimed: U256,
funder: TopUpFunder,
}
impl InFlightTopUp {
fn new(fut: SharedTopUp, requested: U256, funder: TopUpFunder) -> Self {
let unclaimed = match funder {
TopUpFunder::Refill => requested,
TopUpFunder::Recovery => U256::ZERO,
};
Self {
fut,
requested,
unclaimed,
funder,
}
}
fn join(&mut self, want: U256, funder: TopUpFunder) -> (SharedTopUp, TopUpClaim) {
let claimed = match funder {
TopUpFunder::Refill => U256::ZERO,
TopUpFunder::Recovery => self.unclaimed.min(want),
};
self.unclaimed = self.unclaimed.saturating_sub(claimed);
let claim = TopUpClaim::Joined {
claimed,
requested: self.requested,
};
(self.fut.clone(), claim)
}
}
fn joined_headroom(claimed: U256, requested: U256, landed: U256) -> U256 {
if landed >= requested {
claimed
} else {
U256::ZERO
}
}
const MAX_TOPUP_CALLS: u32 = 3;
async fn top_up_at_least<F, Fut>(
pool_id: PoolId,
additional: U256,
mut join_or_spawn: F,
) -> Result<TopUpLanded>
where
F: FnMut(U256) -> (Fut, TopUpClaim),
Fut: std::future::Future<Output = TopUpOutcome>,
{
let mut added = U256::ZERO;
let mut new_deposit = None;
let mut last_err = None;
for _ in 0..MAX_TOPUP_CALLS {
let (fut, claim) = join_or_spawn(additional.saturating_sub(added));
let landed = match (fut.await, claim) {
(Ok(landed), TopUpClaim::Spawned) => {
added = added.saturating_add(landed.added);
let out = TopUpLanded {
new_deposit: landed.new_deposit,
added,
};
if added < additional {
warn_short_top_up(pool_id, additional, out, "our own topUp credited less");
}
return Ok(out);
}
(Ok(landed), TopUpClaim::Joined { claimed, requested }) => {
added = added.saturating_add(joined_headroom(claimed, requested, landed.added));
TopUpLanded {
new_deposit: landed.new_deposit,
added,
}
}
(Err(err), claim) => {
last_err = Some(retryable_join_error(pool_id, additional, err, claim)?);
continue;
}
};
if added >= additional {
return Ok(landed);
}
warn_short_top_up(pool_id, additional, landed, "joined topUp covered less");
new_deposit = Some(landed.new_deposit);
}
match (new_deposit, last_err) {
(Some(new_deposit), _) => {
let out = TopUpLanded { new_deposit, added };
warn_short_top_up(pool_id, additional, out, "funding calls ran out");
Ok(out)
}
(None, Some(err)) => {
let failed =
anyhow::anyhow!("recovery top-up failed: every joined topUp failed, last: {err:#}");
Err(if err.downcast_ref::<PoolNotOpen>().is_some() {
failed.context(PoolNotOpen)
} else {
failed
})
}
(None, None) => Err(anyhow::anyhow!(
"recovery top-up made no funding call (MAX_TOPUP_CALLS is zero)"
)),
}
}
fn warn_short_top_up(pool_id: PoolId, requested: U256, landed: TopUpLanded, why: &str) {
warn!(
%pool_id,
%requested,
landed = %landed.added,
new_deposit = %landed.new_deposit,
why,
"recovery top-up is short of the requested amount"
);
}
fn retryable_join_error(
pool_id: PoolId,
requested: U256,
err: Arc<anyhow::Error>,
claim: TopUpClaim,
) -> Result<Arc<anyhow::Error>> {
if claim == TopUpClaim::Spawned || err.downcast_ref::<EscrowedUntracked>().is_some() {
let failed = anyhow::anyhow!("recovery top-up failed: {err:#}");
return Err(if err.downcast_ref::<PoolNotOpen>().is_some() {
failed.context(PoolNotOpen)
} else {
failed
});
}
warn!(
%pool_id,
%requested,
error = %sanitize_err_chain(&err),
"recovery top-up: the joined topUp failed; funding with our own topUp"
);
Ok(err)
}
fn rehydrate_open_error(err: &Arc<anyhow::Error>) -> anyhow::Error {
let mut rebuilt = anyhow::anyhow!("{err:#}");
if err.downcast_ref::<OpenReported>().is_some() {
rebuilt = rebuilt.context(OpenReported);
}
if err.downcast_ref::<LocalPullFault>().is_some() {
rebuilt = rebuilt.context(LocalPullFault);
}
if let Some(reason) = err.downcast_ref::<decdn_incentive::PoolOpenFailureReason>() {
rebuilt = rebuilt.context(*reason);
}
rebuilt
}
struct SlotGuard<T> {
slot: Arc<Mutex<Option<T>>>,
}
impl<T> Drop for SlotGuard<T> {
fn drop(&mut self) {
*self
.slot
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = None;
}
}
struct FundingHandles<P: Provider + Clone + 'static> {
contract: PaymentPool::PaymentPoolInstance<P>,
store: Arc<dyn BuyerPoolStore>,
rpc: P,
token: Address,
owner: Address,
payment_pool_addr: Address,
metrics: Arc<Metrics>,
}
async fn top_up_recovering_allowance<T, A, AFut, R, RFut>(
attempt: A,
recover_allowance: R,
) -> Result<T>
where
A: Fn() -> AFut,
AFut: std::future::Future<Output = Result<T>>,
R: FnOnce() -> RFut,
RFut: std::future::Future<Output = Result<()>>,
{
match attempt().await {
Ok(out) => Ok(out),
Err(err)
if err
.downcast_ref::<decdn_client::buyer_pool::AllowanceShortfall>()
.is_some() =>
{
recover_allowance().await?;
attempt().await
}
Err(err) => Err(err),
}
}
async fn fund_pool<P: Provider + Clone + 'static>(
handles: &FundingHandles<P>,
pool_id: PoolId,
additional: U256,
) -> Result<TopUpLanded> {
let ToppedUpPool { credited, tx, .. } = match top_up_recovering_allowance(
|| pool_top_up(&handles.contract, handles.owner, pool_id, additional),
|| {
ensure_allowance(
&handles.rpc,
handles.token,
handles.owner,
handles.payment_pool_addr,
None,
)
},
)
.await
{
Ok(topped_up) => topped_up,
Err(err) => {
warn!(
%pool_id,
%additional,
error = %sanitize_err_chain(&err),
"buyer top-up: topUp failed; pool not topped up"
);
handles.metrics.buyer_topup_failure();
return Err(err);
}
};
let effect = topped_up_effect(pool_id, credited);
match grade_deposit_credit(
handles.store.add_deposit(handles.owner, pool_id, credited),
&effect,
tx,
) {
Ok(new_deposit) => {
handles.metrics.buyer_topup_ok();
Ok(TopUpLanded {
new_deposit,
added: credited,
})
}
Err(err) => {
error!(
%pool_id,
%credited,
%tx,
error = %format_args!("{err:#}"),
"buyer top-up: topUp landed on-chain but the local pool row could not be \
credited; the deposit is ESCROWED AND UNTRACKED — reconcile against the chain"
);
handles.metrics.buyer_topup_failure();
Err(err.context(EscrowedUntracked))
}
}
}
fn refill_decision(deposit: U256, spent: U256, working_deposit: U256) -> U256 {
let low_water = working_deposit / U256::from(LOW_WATER_DIVISOR);
refill_amount(deposit, spent, working_deposit, low_water)
}
fn pin_ctx(
state: &BuyerPoolState,
signer: &Arc<PrivateKeySigner>,
voucher_domain: &Eip712Domain,
provider_addr: Address,
) -> Result<PoolContext> {
let lane = LaneKey {
pool_id: state.pool_id,
signer: signer.address(),
provider: provider_addr,
};
let (prior_bytes, prior_amount) = state
.lane_progress(lane)
.map_or((U256::ZERO, U256::ZERO), |p| (p.last_bytes, p.last_amount));
self_owned_lane_ctx(
state,
signer,
voucher_domain,
provider_addr,
prior_bytes,
prior_amount,
)
}
pub struct BuyerPoolService<P: Provider + Clone + 'static> {
contract: PaymentPool::PaymentPoolInstance<P>,
store: Arc<dyn BuyerPoolStore>,
signer: Arc<PrivateKeySigner>,
deployment: Deployment,
voucher_domain: Eip712Domain,
token: Address,
owner: Address,
working_deposit: U256,
open_in_flight: Arc<Mutex<Option<SharedOpen>>>,
open_hold: OpenHold,
topup_in_flight: Arc<Mutex<Option<InFlightTopUp>>>,
seed_slots: Mutex<HashMap<LaneKey, SeedSlot>>,
metrics: Arc<Metrics>,
_reclaimer: AbortOnDropHandle<()>,
}
impl<P: Provider + Clone + 'static> BuyerPoolService<P> {
#[allow(clippy::too_many_arguments)]
pub async fn bootstrap(
provider: P,
deployment: Deployment,
owner: Address,
store: Arc<dyn BuyerPoolStore>,
signer: Arc<PrivateKeySigner>,
working_deposit: U256,
ensure_max_approval: bool,
metrics: Arc<Metrics>,
) -> Result<Self> {
let payment_pool_addr = deployment.payment_pool;
let contract = PaymentPool::new(payment_pool_addr, provider.clone());
let token = contract
.usdc()
.call()
.await
.with_context(|| format!("PaymentPool.usdc() self-check at {payment_pool_addr}"))?;
if ensure_max_approval {
ensure_allowance(&provider, token, owner, payment_pool_addr, None).await?;
}
let load = store.load_all().context("hydrate persisted buyer pools")?;
metrics.buyer_pool_store_skipped_undecodable_records(load.skipped.len());
let reconciled = reconcile_with_chain(
&contract,
deployment,
&store,
owner,
token,
ReconcileRun::Bootstrap,
&metrics,
)
.await;
info!(
%payment_pool_addr,
chain_id = deployment.chain_id,
%token,
%owner,
tracked = load.pools.len(),
?reconciled,
"BuyerPoolService bootstrap complete"
);
publish_buyer_wallet_usdc(&provider, token, owner, &metrics).await;
let reclaimer = tokio::spawn(reclaim_loop(
contract.clone(),
Arc::clone(&store),
owner,
token,
Arc::clone(&metrics),
));
Ok(Self {
contract,
store,
signer,
deployment,
voucher_domain: deployment.voucher_domain(),
token,
owner,
working_deposit,
open_in_flight: Arc::new(Mutex::new(None)),
open_hold: OpenHold::new(),
topup_in_flight: Arc::new(Mutex::new(None)),
seed_slots: Mutex::new(HashMap::new()),
metrics,
_reclaimer: AbortOnDropHandle::new(reclaimer),
})
}
fn reuse_or_report(&self) -> Result<Option<BuyerPoolState>> {
let row = self.store.get_by_owner(self.owner).map_err(|err| {
self.metrics.node_pull_pool_open_failure();
error!(
error = %format_args!("{err:#}"),
"buyer pool store read failed; this node can neither open nor reuse its \
payment pool until the store recovers"
);
anyhow::Error::new(err)
.context("look up the node's buyer pool")
.context(OpenReported)
.context(LocalPullFault)
})?;
let configured = self.deployment;
Ok(row.filter(|state| {
let ours = state.is_on(configured);
if !ours {
warn!(
pool_id = %state.pool_id,
foreign_payment_pool = %state.deployment.payment_pool,
foreign_chain_id = state.deployment.chain_id,
configured_payment_pool = %configured.payment_pool,
configured_chain_id = configured.chain_id,
"ignoring a tracked buyer pool from another PaymentPool deployment; \
opening a fresh pool instead of paying against it"
);
}
ours
}))
}
pub async fn open_or_reuse_pool(
&self,
provider_addr: Address,
budget: Duration,
) -> Result<PoolContext> {
if let Some(state) = self.reuse_or_report()? {
self.spawn_refill_if_low(&state);
let state = self.reseed_lane_from_chain(state, provider_addr).await?;
return pin_ctx(&state, &self.signer, &self.voucher_domain, provider_addr);
}
let open = self.unless_held(self.join_or_spawn_open(None));
match tokio::time::timeout(budget, open).await {
Err(_) => Err(anyhow::Error::new(PoolOpenPending { waited: budget })),
Ok(None) => Err(self.held_open_error()),
Ok(Some(Err(err))) => Err(rehydrate_open_error(&err)),
Ok(Some(Ok(()))) => {
let state = self.reuse_or_report()?.ok_or_else(|| {
self.metrics.node_pull_pool_open_failure();
error!(
"buyer pool opened on-chain but is not present in the store — a deposit \
is escrowed against a row we cannot see"
);
anyhow::anyhow!(
"buyer pool opened but is not present in the store (store/chain disagree?)"
)
.context(OpenReported)
.context(LocalPullFault)
})?;
pin_ctx(&state, &self.signer, &self.voucher_domain, provider_addr)
}
}
}
async fn reseed_lane_from_chain(
&self,
state: BuyerPoolState,
provider_addr: Address,
) -> Result<BuyerPoolState> {
let lane = LaneKey {
pool_id: state.pool_id,
signer: self.signer.address(),
provider: provider_addr,
};
if state.lane_progress(lane).is_some() {
return Ok(state);
}
let turn = self.seed_turn(lane);
let mut shared_read = turn.slot.lock().await;
let state = self.reread_before_seed(lane.pool_id)?;
let lane = LaneKey {
pool_id: state.pool_id,
..lane
};
if state.lane_progress(lane).is_some() {
return Ok(state);
}
let onchain = if let Some((_, onchain)) = shared_read
.as_ref()
.filter(|(read_for, _)| *read_for == lane)
{
onchain.clone()
} else {
let onchain = self.read_watermark(lane).await?;
*shared_read = Some((lane, onchain.clone()));
onchain
};
if onchain.amount == 0 && onchain.bytesDelivered == 0 {
debug!(
pool_id = %state.pool_id, %provider_addr,
"lane has no on-chain watermark; resuming it from zero"
);
return Ok(state);
}
self.persist_lane_seed(lane, &onchain)?;
Ok(self.pin_seeded_state(state, lane, &onchain))
}
fn seed_turn(&self, lane: LaneKey) -> SeedTurn<'_> {
let mut slots = self
.seed_slots
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
SeedTurn {
slots: &self.seed_slots,
lane,
slot: Arc::clone(slots.entry(lane).or_default()),
}
}
async fn read_watermark(&self, lane: LaneKey) -> Result<PaymentPool::Lane> {
let call = self
.contract
.getWatermark(lane.pool_id, lane.signer, lane.provider);
match tokio::time::timeout(WATERMARK_READ_TIMEOUT, call.call()).await {
Ok(Ok(onchain)) => Ok(onchain),
Ok(Err(err)) => {
let shown = sanitize_error_sources(&err);
Err(self.unread_watermark(lane, &shown, anyhow::Error::new(err)))
}
Err(elapsed) => {
let shown = format!("no answer within {WATERMARK_READ_TIMEOUT:?}");
Err(self.unread_watermark(lane, &shown, anyhow::Error::new(elapsed)))
}
}
}
fn reread_before_seed(&self, pool_id: PoolId) -> Result<BuyerPoolState> {
let row = self
.reuse_or_report()
.inspect_err(|_| self.metrics.buyer_lane_seed_failure())
.context("re-read the buyer pool before seeding a lane")?;
row.ok_or_else(|| {
warn!(
%pool_id,
"the buyer pool row vanished while a lane waited to seed; refusing the pull"
);
self.refuse_seed(anyhow::anyhow!(
"buyer pool {pool_id} vanished before its lane could be seeded"
))
})
}
fn refuse_seed(&self, err: anyhow::Error) -> anyhow::Error {
self.metrics.buyer_lane_seed_failure();
self.metrics.node_pull_pool_open_failure();
err.context(OpenReported).context(LocalPullFault)
}
fn unread_watermark(&self, lane: LaneKey, shown: &str, err: anyhow::Error) -> anyhow::Error {
error!(
pool_id = %lane.pool_id, provider_addr = %lane.provider, error = %shown,
"could not read this lane's on-chain watermark; refusing the pull rather than \
resuming the lane from zero, which would strand it below the watermark \
permanently"
);
self.refuse_seed(err.context("read the lane's on-chain watermark"))
}
fn pin_seeded_state(
&self,
state: BuyerPoolState,
lane: LaneKey,
onchain: &PaymentPool::Lane,
) -> BuyerPoolState {
match self.store.get_by_owner(self.owner) {
Ok(Some(seeded)) => seeded,
Ok(None) => {
warn!(
pool_id = %lane.pool_id,
"the buyer pool row vanished between persisting a lane seed and \
re-reading it; pinning the seed from memory"
);
Self::apply_seed_in_memory(state, lane, onchain)
}
Err(err) => {
warn!(
pool_id = %lane.pool_id,
error = %format_args!("{err:#}"),
"could not re-read the buyer pool after seeding a lane; pinning the \
seed from memory"
);
Self::apply_seed_in_memory(state, lane, onchain)
}
}
}
fn apply_seed_in_memory(
mut state: BuyerPoolState,
lane: LaneKey,
onchain: &PaymentPool::Lane,
) -> BuyerPoolState {
if let Err(err) = state.seed_lane(
lane,
U256::from(onchain.bytesDelivered),
U256::from(onchain.amount),
) {
debug!(error = %err, "in-memory lane seed did not advance the snapshot");
}
state
}
#[allow(clippy::cognitive_complexity)]
fn persist_lane_seed(&self, lane: LaneKey, onchain: &PaymentPool::Lane) -> Result<()> {
match self.store.seed_progress(
self.owner,
lane.pool_id,
lane,
U256::from(onchain.bytesDelivered),
U256::from(onchain.amount),
) {
Ok(AdvanceOutcome::Advanced) => {
info!(
pool_id = %lane.pool_id,
provider_addr = %lane.provider,
amount = onchain.amount,
bytes = onchain.bytesDelivered,
"seeded a lane from its on-chain watermark; this node had no local record \
of having paid on it"
);
Ok(())
}
Ok(AdvanceOutcome::Regressed(err)) => {
debug!(
pool_id = %lane.pool_id, provider_addr = %lane.provider, error = %err,
"the committed lane is already at or beyond the chain watermark"
);
Ok(())
}
Ok(AdvanceOutcome::PoolMismatch) => {
info!(
pool_id = %lane.pool_id, provider_addr = %lane.provider,
"a newer pool replaced the buyer row while a lane seeded; pinning the \
replacement"
);
Ok(())
}
Ok(AdvanceOutcome::UnknownPool) => {
warn!(
pool_id = %lane.pool_id, provider_addr = %lane.provider,
"the buyer pool row vanished while a lane seed was persisted; refusing \
the pull"
);
Err(self.refuse_seed(anyhow::anyhow!(
"buyer pool {} vanished before its lane seed could be persisted",
lane.pool_id
)))
}
Err(err) => {
error!(
pool_id = %lane.pool_id, provider_addr = %lane.provider,
error = %format_args!("{err:#}"),
"could not persist a lane seed; refusing the pull rather than resuming \
the lane from zero"
);
Err(self.refuse_seed(
anyhow::Error::new(err).context("persist the lane's on-chain watermark"),
))
}
}
}
fn spawn_refill_if_low(&self, state: &BuyerPoolState) {
let additional = refill_decision(state.deposit, state.pool_spend(), self.working_deposit);
if additional.is_zero() {
return; }
info!(
pool_id = %state.pool_id,
%additional,
"buyer refill: reused pool below its low-water mark; funding toward the working \
deposit (#1146)"
);
drop(self.join_or_spawn_topup(state.pool_id, additional, TopUpFunder::Refill));
}
fn funding_handles(&self) -> FundingHandles<P> {
FundingHandles {
contract: self.contract.clone(),
store: Arc::clone(&self.store),
rpc: self.contract.provider().clone(),
token: self.token,
owner: self.owner,
payment_pool_addr: *self.contract.address(),
metrics: Arc::clone(&self.metrics),
}
}
fn join_or_spawn_topup(
&self,
pool_id: PoolId,
additional: U256,
funder: TopUpFunder,
) -> (SharedTopUp, TopUpClaim) {
let mut slot = self
.topup_in_flight
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(existing) = slot.as_mut() {
debug!(%pool_id, "joining a pool topUp already in flight");
return existing.join(additional, funder);
}
let handles = self.funding_handles();
let guard = SlotGuard {
slot: Arc::clone(&self.topup_in_flight),
};
let task = tokio::spawn(async move {
let _guard = guard;
fund_pool(&handles, pool_id, additional)
.await
.map_err(Arc::new)
});
let fut: BoxFuture<'static, TopUpOutcome> = Box::pin(async move {
task.await.unwrap_or_else(|join_err| {
Err(Arc::new(anyhow::anyhow!(
"buyer pool topUp task failed: {join_err}"
)))
})
});
let shared = fut.shared();
*slot = Some(InFlightTopUp::new(shared.clone(), additional, funder));
(shared, TopUpClaim::Spawned)
}
fn held_open_error(&self) -> anyhow::Error {
self.metrics.node_pull_pool_open_failure();
debug!("the buyer pool open is held for an unconfirmed openPool; refusing at once");
anyhow::Error::new(OpenHeld)
.context(OpenReported)
.context(LocalPullFault)
}
async fn unless_held(&self, open: SharedOpen) -> Option<OpenOutcome> {
tokio::select! {
biased;
outcome = open => Some(outcome),
() = self.open_hold.wait_held() => None,
}
}
fn join_or_spawn_open(&self, replacing: Option<PoolId>) -> SharedOpen {
let mut slot = self
.open_in_flight
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(existing) = slot.as_ref() {
debug!("joining an openPool already in flight");
return existing.clone();
}
let contract = self.contract.clone();
let store = Arc::clone(&self.store);
let open = OpenRequest {
signer: Arc::clone(&self.signer),
deployment: self.deployment,
token: self.token,
owner: self.owner,
deposit: self.working_deposit,
};
let hold = self.open_hold.clone();
let metrics = Arc::clone(&self.metrics);
let guard = SlotGuard {
slot: Arc::clone(&self.open_in_flight),
};
let task = tokio::spawn(async move {
let _guard = guard;
run_open(&contract, &store, open, replacing, &hold, &metrics)
.await
.map_err(Arc::new)
});
let join_metrics = Arc::clone(&self.metrics);
let fut: BoxFuture<'static, OpenOutcome> = Box::pin(async move {
task.await.unwrap_or_else(|join_err| {
join_metrics.node_pull_pool_open_failure();
error!(%join_err, "buyer pool open task did not run to completion");
Err(Arc::new(
anyhow::anyhow!("buyer pool open task failed: {join_err}")
.context(OpenReported)
.context(LocalPullFault),
))
})
});
let shared = fut.shared();
*slot = Some(shared.clone());
shared
}
pub fn record_progress(
&self,
provider_addr: Address,
pool_id: PoolId,
write: ProgressWrite,
) -> Result<()> {
let lane = LaneKey {
pool_id,
signer: self.signer.address(),
provider: provider_addr,
};
let totals = write.totals();
let outcome = write
.apply(self.store.as_ref(), WatermarkRow::Owned(self.owner), lane)
.with_context(|| match write {
ProgressWrite::Rebase { anchor, .. } => format!(
"rebase buyer pool lane progress down to the upstream watermark \
(amount {}, bytes {}), then to (amount {}, bytes {})",
anchor.last_amount, anchor.last_bytes, totals.last_amount, totals.last_bytes
),
ProgressWrite::Advance { .. } => format!(
"advance buyer pool lane progress to (amount {}, bytes {})",
totals.last_amount, totals.last_bytes
),
_ => format!(
"record buyer pool lane progress at (amount {}, bytes {})",
totals.last_amount, totals.last_bytes
),
});
match outcome? {
AdvanceOutcome::Advanced => Ok(()),
AdvanceOutcome::UnknownPool => {
anyhow::bail!("record_progress for unknown pool {pool_id}")
}
AdvanceOutcome::PoolMismatch => {
self.metrics.node_pull_progress_dropped();
warn!(
provider = %provider_addr,
%pool_id,
"record_progress: the owner's pool was replaced by a newer open; \
skipping stale progress write"
);
Ok(())
}
AdvanceOutcome::Regressed(_err) => {
self.metrics.node_pull_progress_superseded();
debug!(
provider = %provider_addr,
%pool_id,
"record_progress: a concurrent settle persisted a higher watermark first; \
ours superseded (benign under the shared ledger)"
);
Ok(())
}
}
}
pub async fn recover_pool(&self, seen_deposit: U256) -> Result<Recovery> {
let state = self
.reuse_or_report()?
.with_context(|| "no buyer pool tracked to recover")?;
if state.deposit > seen_deposit {
return Ok(Recovery::ToppedUp(state.deposit));
}
if let Some(in_flight) = self.recovery_in_flight() {
return match in_flight.await {
Ok(landed) => Ok(Recovery::ToppedUp(landed.new_deposit)),
Err(err) if err.downcast_ref::<PoolNotOpen>().is_some() => {
self.replace_pool(state.pool_id).await
}
Err(err) => Err(anyhow::anyhow!(
"a sibling's recovery top-up failed: {err:#}"
)),
};
}
if self.working_deposit.is_zero() {
return Ok(Recovery::Unavailable);
}
let remaining = state.deposit.saturating_sub(state.pool_spend());
let additional = self.working_deposit.saturating_sub(remaining);
if additional.is_zero() {
return self.settle_or_replace(state.pool_id, state.deposit).await;
}
match self.top_up_pool_by(additional).await {
Ok(landed) if landed.added.is_zero() => Ok(Recovery::Unavailable),
Ok(landed) => Ok(Recovery::ToppedUp(landed.new_deposit)),
Err(err) if err.downcast_ref::<PoolNotOpen>().is_some() => {
self.replace_pool(state.pool_id).await
}
Err(err) => Err(err),
}
}
async fn replace_pool(&self, closed: PoolId) -> Result<Recovery> {
info!(
pool_id = %closed,
"buyer pool no longer accepts funds; opening a new pool"
);
let opened = self.open_replacement(closed).await?;
Ok(Recovery::Replaced(PoolReplaced { closed, opened }))
}
async fn settle_or_replace(&self, pool_id: PoolId, deposit: U256) -> Result<Recovery> {
if pool_accepts_funds(&self.contract, self.owner, pool_id).await {
info!(
%pool_id,
%deposit,
"buyer pool already holds its working deposit; waiting for the upstreams to \
see it"
);
return Ok(Recovery::ToppedUp(deposit));
}
info!(
%pool_id,
"buyer pool holds its working deposit but accepts no more funds; opening a new pool"
);
let opened = self.open_replacement(pool_id).await?;
Ok(Recovery::Replaced(PoolReplaced {
closed: pool_id,
opened,
}))
}
fn recovery_in_flight(&self) -> Option<SharedTopUp> {
self.topup_in_flight
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.filter(|in_flight| in_flight.funder == TopUpFunder::Recovery)
.map(|in_flight| in_flight.fut.clone())
}
pub fn pool_deposit(&self) -> Option<U256> {
self.store
.get_by_owner(self.owner)
.ok()
.flatten()
.map(|state| state.deposit)
}
async fn open_replacement(&self, closed: PoolId) -> Result<PoolId> {
let Some(outcome) = self
.unless_held(self.join_or_spawn_open(Some(closed)))
.await
else {
return Err(self.held_open_error());
};
outcome.map_err(|err| rehydrate_open_error(&err))?;
let state = self
.reuse_or_report()?
.with_context(|| "the replacement buyer pool is not in the store")?;
if state.pool_id == closed {
anyhow::bail!("no replacement opened for closed buyer pool {closed}");
}
Ok(state.pool_id)
}
pub async fn top_up_pool_by(&self, additional: U256) -> Result<TopUpLanded> {
let state = self
.reuse_or_report()?
.with_context(|| "no buyer pool tracked to top up")?;
if additional.is_zero() {
return Ok(TopUpLanded {
new_deposit: state.deposit,
added: U256::ZERO,
});
}
let landed = top_up_at_least(state.pool_id, additional, |amount| {
self.join_or_spawn_topup(state.pool_id, amount, TopUpFunder::Recovery)
})
.await?;
info!(
pool_id = %state.pool_id,
%additional,
added = %landed.added,
new_deposit = %landed.new_deposit,
"recovery top-up: funded the buyer pool (#1530)"
);
Ok(landed)
}
pub async fn sweep_reclaimable_once(&self) {
reclaim_once(&self.contract, &self.store, self.owner, &self.metrics).await;
}
}
impl<P: Provider + Clone + 'static> std::fmt::Debug for BuyerPoolService<P> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BuyerPoolService")
.field("address", self.contract.address())
.field("owner", &self.owner)
.finish_non_exhaustive()
}
}
#[async_trait::async_trait]
pub trait PoolOpener: Send + Sync + std::fmt::Debug {
async fn open_or_reuse_pool(
&self,
provider_addr: Address,
budget: Duration,
) -> Result<PoolContext>;
fn record_progress(
&self,
provider_addr: Address,
pool_id: PoolId,
write: ProgressWrite,
) -> Result<()>;
async fn recover_pool(&self, _seen_deposit: U256) -> Result<Recovery> {
Ok(Recovery::Unavailable)
}
fn pool_deposit(&self) -> Option<U256> {
None
}
}
#[async_trait::async_trait]
impl<P: Provider + Clone + 'static> PoolOpener for BuyerPoolService<P> {
async fn open_or_reuse_pool(
&self,
provider_addr: Address,
budget: Duration,
) -> Result<PoolContext> {
BuyerPoolService::open_or_reuse_pool(self, provider_addr, budget).await
}
fn record_progress(
&self,
provider_addr: Address,
pool_id: PoolId,
write: ProgressWrite,
) -> Result<()> {
BuyerPoolService::record_progress(self, provider_addr, pool_id, write)
}
async fn recover_pool(&self, seen_deposit: U256) -> Result<Recovery> {
BuyerPoolService::recover_pool(self, seen_deposit).await
}
fn pool_deposit(&self) -> Option<U256> {
BuyerPoolService::pool_deposit(self)
}
}
macro_rules! warn_at_bootstrap {
($run:expr, $($arg:tt)+) => {
if $run.is_bootstrap() {
warn!($($arg)+);
} else {
debug!($($arg)+);
}
};
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ReconcileRun {
Bootstrap,
Settle {
block: u64,
},
}
impl ReconcileRun {
const fn at(self) -> BlockId {
match self {
Self::Bootstrap => BlockId::latest(),
Self::Settle { block } => BlockId::number(block),
}
}
const fn is_bootstrap(self) -> bool {
matches!(self, Self::Bootstrap)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Reconciled {
Adopted(PoolId),
KeptTracked,
NoneHeld,
Unknown,
Unrecorded(PoolId),
}
async fn reconcile_with_chain<P: Provider + Clone>(
contract: &PaymentPool::PaymentPoolInstance<P>,
deployment: Deployment,
store: &Arc<dyn BuyerPoolStore>,
owner: Address,
token: Address,
run: ReconcileRun,
metrics: &Arc<Metrics>,
) -> Reconciled {
let tracked: Option<PoolId> = match adoption_applies(store, owner, deployment, run) {
AdoptionCheck::Unknown => {
if run.is_bootstrap() {
metrics.buyer_pool_adoption_failure();
}
return Reconciled::Unknown;
}
AdoptionCheck::Applies => None,
AdoptionCheck::Foreign { pool_id, was_on } => {
if !drop_foreign_row(store, owner, pool_id, was_on, deployment) {
return Reconciled::Unknown;
}
None
}
AdoptionCheck::AlreadyTracked(pool_id) => Some(pool_id),
};
let Some(pools) = enumerate_open_pools(contract, owner, tracked.is_none(), run, metrics).await
else {
return Reconciled::Unknown;
};
let candidate = match tracked {
Some(pool_id) => match reconcile_tracked(&pools, store, owner, pool_id, run) {
TrackedOutcome::Keep => return Reconciled::KeptTracked,
TrackedOutcome::Undecided => return Reconciled::Unknown,
TrackedOutcome::Replace(candidate) => candidate,
},
None => pools.newest_solvent(),
};
let Some((pool_id, pool)) = candidate else {
return if pools.unreadable.is_empty() {
Reconciled::NoneHeld
} else {
Reconciled::Unknown
};
};
let state = BuyerPoolState::adopt(
pool_id,
deployment,
owner,
token,
U256::from(pool.deposit),
U256::from(pool.totalRedeemed),
);
if let Err(err) = store.record(&state) {
if run.is_bootstrap() {
metrics.buyer_pool_adoption_failure();
}
warn!(
%pool_id,
error = %format_args!("{err:#}"),
"could not persist the adopted pool; this node does not track it"
);
return Reconciled::Unrecorded(pool_id);
}
report_stranded_pools(&pools.recoverable_beside(pool_id), pools.unreadable.len());
info!(
%pool_id,
deposit = pool.deposit,
remaining = pool.deposit.saturating_sub(pool.totalRedeemed),
"adopted this node's existing on-chain payment pool; not opening a second one"
);
Reconciled::Adopted(pool_id)
}
#[allow(clippy::cognitive_complexity)]
async fn enumerate_open_pools<P: Provider + Clone>(
contract: &PaymentPool::PaymentPoolInstance<P>,
owner: Address,
adopting: bool,
run: ReconcileRun,
metrics: &Arc<Metrics>,
) -> Option<OwnedPools> {
let ids = match enumerate_owned_pools_at(contract, owner, run.at()).await {
Ok(ids) => ids,
Err(err) if adopting => {
if run.is_bootstrap() {
metrics.buyer_pool_adoption_failure();
}
warn_at_bootstrap!(
run,
error = %sanitize_err_chain(&err),
"could not enumerate this node's on-chain pools; a pool it already owns \
stays unadopted and the next miss opens a fresh one"
);
return None;
}
Err(err) => {
warn_at_bootstrap!(
run,
error = %sanitize_err_chain(&err),
"could not enumerate this node's on-chain pools; a deposit stranded beside \
the pool it is using would go unreported this pass"
);
return None;
}
};
Some(OwnedPools::walk(contract, ids, run).await)
}
enum TrackedOutcome {
Keep,
Undecided,
Replace(Option<(PoolId, PaymentPool::Pool)>),
}
fn reconcile_tracked(
pools: &OwnedPools,
store: &Arc<dyn BuyerPoolStore>,
owner: Address,
pool_id: PoolId,
run: ReconcileRun,
) -> TrackedOutcome {
match pools.tracked_status(pool_id) {
TrackedStatus::Open => {
report_stranded_pools(&pools.recoverable_beside(pool_id), pools.unreadable.len());
TrackedOutcome::Keep
}
TrackedStatus::Unknown => {
warn_at_bootstrap!(
run,
%pool_id,
"could not read the tracked buyer pool's on-chain status; keeping the row. A \
pool this node owns must not be dropped because one read failed — the next \
reconciliation asks again"
);
TrackedOutcome::Undecided
}
TrackedStatus::NotOpen => {
if drop_stale_row(store, owner, pool_id) {
TrackedOutcome::Replace(pools.newest_solvent())
} else {
TrackedOutcome::Undecided
}
}
}
}
fn drop_foreign_row(
store: &Arc<dyn BuyerPoolStore>,
owner: Address,
pool_id: PoolId,
was_on: Deployment,
configured: Deployment,
) -> bool {
warn!(
%pool_id,
foreign_payment_pool = %was_on.payment_pool,
foreign_chain_id = was_on.chain_id,
configured_payment_pool = %configured.payment_pool,
configured_chain_id = configured.chain_id,
"the tracked buyer pool belongs to a different PaymentPool deployment; dropping the \
stale row. Its deposit, if any, is recoverable only against \
`foreign_payment_pool`, which is the last record this node keeps of it"
);
if let Err(err) = store.forget_if_pool(owner, pool_id) {
warn!(
%pool_id,
foreign_payment_pool = %was_on.payment_pool,
foreign_chain_id = was_on.chain_id,
error = %format_args!("{err:#}"),
"could not drop the foreign buyer pool row; this node keeps reusing a pool on a \
contract that has never heard of it, and its vouchers fund nothing, until the \
store recovers"
);
return false;
}
true
}
fn drop_stale_row(store: &Arc<dyn BuyerPoolStore>, owner: Address, pool_id: PoolId) -> bool {
warn!(
%pool_id,
"the tracked buyer pool is no longer open on chain; dropping the stale row so this \
node stops pinning its pulls to a pool that can fund no voucher"
);
if let Err(err) = store.forget_if_pool(owner, pool_id) {
warn!(
%pool_id,
error = %format_args!("{err:#}"),
"could not drop the stale buyer pool row; this node keeps reusing a pool whose \
vouchers stop redeeming at its dispute deadline, until the store recovers"
);
return false;
}
true
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum AdoptionCheck {
Applies,
Foreign {
pool_id: PoolId,
was_on: Deployment,
},
AlreadyTracked(PoolId),
Unknown,
}
fn adoption_applies(
store: &Arc<dyn BuyerPoolStore>,
owner: Address,
deployment: Deployment,
run: ReconcileRun,
) -> AdoptionCheck {
match store.get_by_owner(owner) {
Ok(None) => AdoptionCheck::Applies,
Ok(Some(state)) if !state.is_on(deployment) => AdoptionCheck::Foreign {
pool_id: state.pool_id,
was_on: state.deployment,
},
Ok(Some(state)) => AdoptionCheck::AlreadyTracked(state.pool_id),
Err(err) => {
warn_at_bootstrap!(
run,
error = %format_args!("{err:#}"),
"buyer pool store read failed before on-chain reconciliation; \
skipping adoption and leaving the open path to report it"
);
AdoptionCheck::Unknown
}
}
}
fn report_stranded_pools(stranded: &[PoolId], unreadable: usize) {
if stranded.is_empty() {
if unreadable > 0 {
warn!(
unreadable,
"could not read every pool this node owns, so this pass cannot say whether it \
holds a stranded deposit; the absence of a stranded-pool warning is not \
evidence there is none"
);
}
return;
}
warn!(
count = stranded.len(),
pools = ?stranded,
unreadable,
"this node owns further open payment pools it is not using; their deposits are \
recoverable with `decdn pool close --pool <id>` then, after the dispute window, \
`decdn pool reclaim --pool <id>`. Name each id: `--all` would close the pool this \
node is using, and is refused on a node's data dir. Neither command clears this \
node's own record — restart it afterwards if you closed the pool it is using"
);
}
struct OwnedPools {
read: Vec<(PoolId, PaymentPool::Pool)>,
unreadable: Vec<PoolId>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TrackedStatus {
Open,
NotOpen,
Unknown,
}
impl OwnedPools {
async fn walk<P: Provider + Clone>(
contract: &PaymentPool::PaymentPoolInstance<P>,
ids: Vec<PoolId>,
run: ReconcileRun,
) -> Self {
let mut read = Vec::new();
let mut unreadable = Vec::new();
for pool_id in ids.into_iter().rev() {
match contract.getPool(pool_id).call().block(run.at()).await {
Ok(pool) => read.push((pool_id, pool)),
Err(err) => {
warn_at_bootstrap!(
run,
%pool_id,
error = %sanitize_error_sources(&err),
"could not read an owned pool's state; this pass cannot say whether it \
is open, so it is neither adopted nor reported as stranded"
);
unreadable.push(pool_id);
}
}
}
Self { read, unreadable }
}
fn tracked_status(&self, pool_id: PoolId) -> TrackedStatus {
self.read.iter().find(|(id, _)| *id == pool_id).map_or(
TrackedStatus::Unknown,
|(_, pool)| {
if matches!(pool.status, PaymentPool::Status::Open) {
TrackedStatus::Open
} else {
TrackedStatus::NotOpen
}
},
)
}
fn open(&self) -> impl Iterator<Item = &(PoolId, PaymentPool::Pool)> {
self.read
.iter()
.filter(|(_, pool)| matches!(pool.status, PaymentPool::Status::Open))
}
fn newest_solvent(&self) -> Option<(PoolId, PaymentPool::Pool)> {
self.open()
.find(|(_, pool)| pool.deposit > pool.totalRedeemed)
.map(|(id, pool)| (*id, pool.clone()))
}
fn recoverable_beside(&self, in_use: PoolId) -> Vec<PoolId> {
self.open()
.filter(|(id, pool)| *id != in_use && pool.deposit > pool.totalRedeemed)
.map(|(id, _)| *id)
.collect()
}
}
#[allow(clippy::cognitive_complexity)]
async fn run_open<P: Provider + Clone>(
contract: &PaymentPool::PaymentPoolInstance<P>,
store: &Arc<dyn BuyerPoolStore>,
open: OpenRequest,
replacing: Option<PoolId>,
hold: &OpenHold,
metrics: &Arc<Metrics>,
) -> Result<()> {
match store.get_by_owner(open.owner) {
Ok(Some(state)) if state.is_on(open.deployment) && Some(state.pool_id) != replacing => {
debug!("a live buyer pool appeared while claiming the open slot; not opening a second");
return Ok(());
}
Ok(_) => {}
Err(err) => {
error!(error = %err, "buyer pool store read failed under the open slot; cannot open a pool");
metrics.node_pull_pool_open_failure();
return Err(anyhow::Error::new(err))
.context("look up the node's buyer pool under the open slot")
.context(OpenReported)
.context(LocalPullFault);
}
}
let landed = match open_pool(
contract,
Arc::clone(&open.signer),
open.deployment,
open.token,
open.owner,
open.deposit,
None,
)
.await
{
Ok(opened) => Ok(Landed::Opened(Box::new(opened))),
Err(err) => match err.downcast_ref::<OpenUnconfirmed>().copied() {
Some(unconfirmed) => {
settle_unconfirmed_open(contract, store, &open, unconfirmed, err, hold, metrics)
.await
}
None => Err(err),
},
};
let opened = match landed {
Ok(Landed::Opened(opened)) => opened,
Ok(Landed::Adopted) => return Ok(()),
Err(err) => {
error!(error = %sanitize_err_chain(&err), "buyer pool open failed");
metrics.node_pull_pool_open_failure();
let reason = err.downcast_ref::<PoolOpenFailureReason>().copied();
if let Some(reason) = reason {
metrics.pool_open_failure_by_reason(reason);
}
let node_wide = matches!(
reason,
Some(PoolOpenFailureReason::InsufficientDeposit | PoolOpenFailureReason::RpcError)
);
let err = err.context(OpenReported);
return Err(if node_wide {
err.context(LocalPullFault)
} else {
err
});
}
};
if let Err(err) = store.record(&opened.state) {
error!(
tx = %opened.tx,
pool_id = %opened.state.pool_id,
error = %err,
"buyer pool opened on-chain but its row could not be persisted; the deposit is \
escrowed but UNTRACKED — reconcile against the tx"
);
metrics.node_pull_pool_open_failure();
return Err(anyhow::Error::new(err))
.context("persist opened buyer pool")
.context(OpenReported)
.context(LocalPullFault);
}
info!(pool_id = %opened.state.pool_id, deposit = %open.deposit, "opened buyer payment pool");
Ok(())
}
struct OpenRequest {
signer: Arc<PrivateKeySigner>,
deployment: Deployment,
token: Address,
owner: Address,
deposit: U256,
}
#[derive(Debug)]
enum Landed {
Opened(Box<OpenedPool>),
Adopted,
}
#[derive(Clone)]
struct OpenHold {
held: Arc<tokio::sync::watch::Sender<bool>>,
}
impl OpenHold {
fn new() -> Self {
Self {
held: Arc::new(tokio::sync::watch::channel(false).0),
}
}
fn hold(&self, metrics: &Arc<Metrics>) -> HeldGuard {
self.held.send_replace(true);
metrics.set_buyer_pool_open_unresolved(true);
HeldGuard {
held: Arc::clone(&self.held),
metrics: Arc::clone(metrics),
}
}
async fn wait_held(&self) {
let mut held = self.held.subscribe();
if held.wait_for(|held| *held).await.is_err() {
std::future::pending::<()>().await;
}
}
}
struct HeldGuard {
held: Arc<tokio::sync::watch::Sender<bool>>,
metrics: Arc<Metrics>,
}
impl Drop for HeldGuard {
fn drop(&mut self) {
self.held.send_replace(false);
self.metrics.set_buyer_pool_open_unresolved(false);
}
}
#[derive(Debug)]
struct OpenHeld;
impl std::fmt::Display for OpenHeld {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(
"the buyer pool open is held for an unconfirmed openPool; this node opens no pool \
until the chain resolves it",
)
}
}
impl std::error::Error for OpenHeld {}
#[derive(Debug)]
enum OpenResolution {
Mined(Box<alloy::rpc::types::TransactionReceipt>),
NonceSpent {
block: u64,
},
Pending,
Queued,
Vacant,
}
async fn settle_unconfirmed_open<P: Provider + Clone>(
contract: &PaymentPool::PaymentPoolInstance<P>,
store: &Arc<dyn BuyerPoolStore>,
open: &OpenRequest,
unconfirmed: OpenUnconfirmed,
err: anyhow::Error,
hold: &OpenHold,
metrics: &Arc<Metrics>,
) -> Result<Landed> {
let _held = hold.hold(metrics);
let started = tokio::time::Instant::now();
let mut settling = Settling {
nonce: unconfirmed.nonce,
txs: unconfirmed.tx.into_iter().collect(),
original: sanitize_err_chain(&err),
started,
next_reconcile: started,
reconcile_backoff: OPEN_RESOLVE_INTERVAL,
};
warn!(
nonce = settling.nonce,
txs = ?settling.txs,
error = %settling.original,
"openPool outcome unknown; holding the open slot until the chain resolves it, so \
no second deposit escrows beside it"
);
let mut ticker = tokio::time::interval(OPEN_RESOLVE_INTERVAL);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut last_warned = settling.started;
let mut last_seen = String::from("no chain read has answered yet");
loop {
ticker.tick().await;
if last_warned.elapsed() >= OPEN_UNRESOLVED_WARN_INTERVAL {
last_warned = tokio::time::Instant::now();
warn!(
nonce = settling.nonce,
txs = ?settling.txs,
elapsed = ?settling.started.elapsed(),
last = %last_seen,
"still holding the open slot for an unconfirmed openPool; this node opens no \
buyer pool until the chain resolves it"
);
}
match settling.tick(contract, store, open, metrics).await {
Tick::Wait(seen) => last_seen = seen,
Tick::Done(landed) => return landed,
}
}
}
struct Settling {
nonce: u64,
txs: Vec<TxHash>,
original: String,
started: tokio::time::Instant,
next_reconcile: tokio::time::Instant,
reconcile_backoff: Duration,
}
enum Tick {
Wait(String),
Done(Result<Landed>),
}
impl Settling {
async fn tick<P: Provider + Clone>(
&mut self,
contract: &PaymentPool::PaymentPoolInstance<P>,
store: &Arc<dyn BuyerPoolStore>,
open: &OpenRequest,
metrics: &Arc<Metrics>,
) -> Tick {
let resolution = match resolve_unconfirmed_open(contract, open.owner, self.nonce, &self.txs)
.await
{
Ok(resolution) => resolution,
Err(err) => {
let seen = format!("a chain read failed: {}", sanitize_err_chain(&err));
debug!(nonce = self.nonce, last = %seen, "could not read the unconfirmed openPool's state");
return Tick::Wait(seen);
}
};
match resolution {
OpenResolution::Pending => Tick::Wait("a pending transaction holds the nonce".into()),
OpenResolution::Queued => {
Tick::Wait("an earlier nonce of this account is not confirmed".into())
}
OpenResolution::Vacant => Tick::Wait(self.resend(contract, open).await),
OpenResolution::Mined(receipt) => {
let tx = receipt.transaction_hash;
info!(
nonce = self.nonce,
%tx,
elapsed = ?self.started.elapsed(),
"unconfirmed openPool resolved: a transaction at its nonce mined"
);
Tick::Done(
opened_from_receipt(
&receipt,
tx,
Arc::clone(&open.signer),
open.deployment,
open.token,
open.owner,
open.deposit,
)
.map(|opened| Landed::Opened(Box::new(opened))),
)
}
OpenResolution::NonceSpent { block } => {
let now = tokio::time::Instant::now();
if now < self.next_reconcile {
return Tick::Wait(format!(
"the nonce is spent at block {block}; the next reconciliation is due in \
{:?}",
self.next_reconcile - now
));
}
let run = ReconcileRun::Settle { block };
let reconciled = reconcile_with_chain(
contract,
open.deployment,
store,
open.owner,
open.token,
run,
metrics,
)
.await;
self.spent(block, reconciled)
}
}
}
async fn resend<P: Provider + Clone>(
&mut self,
contract: &PaymentPool::PaymentPoolInstance<P>,
open: &OpenRequest,
) -> String {
match send_open_pool(contract, open.owner, open.deposit, self.nonce).await {
Ok(tx) => {
info!(
nonce = self.nonce,
%tx,
"no transaction holds the unconfirmed openPool's nonce; re-sent openPool \
at that nonce"
);
if self.txs.len() >= MAX_OPEN_TXS {
self.txs.remove(0);
}
self.txs.push(tx);
format!("re-sent openPool at the nonce as {tx}")
}
Err(err) => {
let seen = format!(
"no transaction holds the nonce, and the re-send failed: {}",
sanitize_err_chain(&err)
);
debug!(nonce = self.nonce, last = %seen, "could not re-send the openPool");
seen
}
}
}
fn spent(&mut self, block: u64, reconciled: Reconciled) -> Tick {
let nonce = self.nonce;
match reconciled {
Reconciled::Adopted(pool_id) => {
info!(
nonce,
block,
%pool_id,
elapsed = ?self.started.elapsed(),
"unconfirmed openPool resolved: its nonce is spent, and the owner's newest \
solvent pool is adopted"
);
Tick::Done(Ok(Landed::Adopted))
}
Reconciled::NoneHeld => {
info!(
nonce,
block,
elapsed = ?self.started.elapsed(),
"unconfirmed openPool resolved: its nonce is spent and the owner holds no \
open pool, so it escrowed nothing"
);
Tick::Done(Err(anyhow::anyhow!(
"the openPool nonce {nonce} is spent at block {block} and this owner holds \
no open pool there, so the open escrowed nothing: {}",
self.original
)
.context(PoolOpenFailureReason::RpcError)))
}
Reconciled::Unrecorded(pool_id) => {
error!(
nonce,
%pool_id,
"the pool an unconfirmed openPool resolved to could not be recorded; the \
deposit is escrowed but UNTRACKED"
);
Tick::Done(Err(anyhow::anyhow!(
"buyer pool {pool_id} could not be recorded after an unconfirmed openPool; \
the deposit is escrowed but untracked"
)
.context(LocalPullFault)))
}
Reconciled::KeptTracked | Reconciled::Unknown => {
self.next_reconcile = tokio::time::Instant::now() + self.reconcile_backoff;
self.reconcile_backoff = self
.reconcile_backoff
.saturating_mul(2)
.min(OPEN_RECONCILE_MAX_BACKOFF);
Tick::Wait(format!(
"the nonce is spent at block {block}, but reconciliation could not answer"
))
}
}
}
}
async fn resolve_unconfirmed_open<P: Provider + Clone>(
contract: &PaymentPool::PaymentPoolInstance<P>,
owner: Address,
nonce: u64,
txs: &[TxHash],
) -> Result<OpenResolution> {
let provider = contract.provider();
if let Some(receipt) = first_receipt(provider, txs).await? {
return Ok(OpenResolution::Mined(receipt));
}
let block = provider
.get_block_number()
.await
.context("read the block number")?;
let confirmed = provider
.get_transaction_count(owner)
.number(block)
.await
.context("read the confirmed nonce")?;
if confirmed > nonce {
if let Some(receipt) = first_receipt(provider, txs).await? {
return Ok(OpenResolution::Mined(receipt));
}
return Ok(OpenResolution::NonceSpent { block });
}
if confirmed < nonce {
return Ok(OpenResolution::Queued);
}
let pending = provider
.get_transaction_count(owner)
.pending()
.await
.context("read the pending nonce")?;
Ok(if pending > nonce {
OpenResolution::Pending
} else {
OpenResolution::Vacant
})
}
async fn first_receipt<P: Provider>(
provider: &P,
txs: &[TxHash],
) -> Result<Option<Box<alloy::rpc::types::TransactionReceipt>>> {
for &tx in txs {
if let Some(receipt) = provider
.get_transaction_receipt(tx)
.await
.context("read an openPool receipt")?
{
return Ok(Some(Box::new(receipt)));
}
}
Ok(None)
}
async fn reclaim_loop<P: Provider + Clone>(
contract: PaymentPool::PaymentPoolInstance<P>,
store: Arc<dyn BuyerPoolStore>,
owner: Address,
token: Address,
metrics: Arc<Metrics>,
) {
let mut ticker = tokio::time::interval(RECLAIM_SWEEP_INTERVAL);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
ticker.tick().await;
reclaim_once(&contract, &store, owner, &metrics).await;
publish_buyer_wallet_usdc(contract.provider(), token, owner, &metrics).await;
}
}
async fn publish_buyer_wallet_usdc<P: Provider>(
provider: &P,
token: Address,
owner: Address,
metrics: &Arc<Metrics>,
) {
match Erc20::new(token, provider).balanceOf(owner).call().await {
Ok(balance) => metrics.set_buyer_wallet_usdc(balance),
Err(err) => {
debug!(%token, %owner, error = %sanitize_error_sources(&err), "could not read the buyer wallet's USDC balance");
}
}
}
async fn head_timestamp<P: Provider>(provider: &P) -> Result<u64> {
let block = provider
.get_block(BlockId::latest())
.await
.context("read the latest block for the reclaim grace-window pre-check")?
.ok_or_else(|| anyhow::anyhow!("no latest block"))?;
Ok(block.header.timestamp)
}
#[allow(clippy::cognitive_complexity)]
async fn reclaim_once<P: Provider + Clone>(
contract: &PaymentPool::PaymentPoolInstance<P>,
store: &Arc<dyn BuyerPoolStore>,
owner: Address,
metrics: &Arc<Metrics>,
) {
let Some(state) = (match store.get_by_owner(owner) {
Ok(state) => state,
Err(err) => {
warn!(error = %err, "reclaim sweep: buyer pool store read failed");
metrics.buyer_reclaim_failure();
return;
}
}) else {
return; };
let pool = match contract.getPool(state.pool_id).call().await {
Ok(pool) => pool,
Err(err) => {
warn!(pool_id = %state.pool_id, error = %sanitize_error_sources(&err), "reclaim sweep: getPool failed");
metrics.buyer_reclaim_failure();
return;
}
};
if matches!(pool.status, PaymentPool::Status::Closed) {
match store.forget_if_pool(owner, state.pool_id) {
Ok(_) => info!(
pool_id = %state.pool_id,
"the buyer pool was already reclaimed; dropped the row"
),
Err(err) => {
warn!(pool_id = %state.pool_id, error = %err, "reclaim sweep: forget of a reclaimed pool failed");
metrics.buyer_reclaim_failure();
}
}
return;
}
if pool.disputeDeadline == 0 {
return;
}
match head_timestamp(contract.provider()).await {
Ok(now) if now < pool.disputeDeadline => return,
Ok(_) => {}
Err(err) => {
debug!(pool_id = %state.pool_id, error = %sanitize_err_chain(&err), "reclaim sweep: chain head read failed; attempting reclaim anyway");
}
}
match contract.reclaim(state.pool_id).send().await {
Ok(pending) => {
let tx = *pending.tx_hash();
match pending.get_receipt().await {
Ok(receipt) if receipt.status() => {
if let Err(err) = store.forget_if_pool(owner, state.pool_id) {
warn!(pool_id = %state.pool_id, %tx, error = %err, "reclaim sweep: forget after reclaim failed");
metrics.buyer_reclaim_failure();
return;
}
info!(pool_id = %state.pool_id, %tx, "reclaimed the buyer pool residual and dropped the row");
}
Ok(_) => {
warn!(pool_id = %state.pool_id, %tx, "reclaim sweep: reclaim reverted");
metrics.buyer_reclaim_failure();
}
Err(err) => {
warn!(pool_id = %state.pool_id, %tx, error = %sanitize_error_sources(&err), "reclaim sweep: reclaim receipt failed");
metrics.buyer_reclaim_failure();
}
}
}
Err(err) => {
warn!(pool_id = %state.pool_id, error = %sanitize_error_sources(&err), "reclaim sweep: reclaim submit failed");
metrics.buyer_reclaim_failure();
}
}
}
#[allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::indexing_slicing,
reason = "workspace anti-panic policy targets runtime code"
)]
#[cfg(test)]
mod tests;