use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use alloy::dyn_abi::Eip712Domain;
use alloy::primitives::{Address, B256, U256};
use alloy::signers::local::PrivateKeySigner;
use anyhow::Result;
use async_trait::async_trait;
use bytes::BytesMut;
use decdn_cache::origin::{FilesystemOrigin, Origin, OriginFetch};
use decdn_cache::{
CacheEngine, CacheMetrics, CircuitBreakerPolicy, Hash, PinnedHashes, RetryPolicy,
};
use decdn_client::LocalPullFault;
use decdn_client::probe::probe_once;
use decdn_client::{PoolContext, Recovery};
use decdn_incentive::{
EPHEMERAL_BINDING_NONCE, LaneKey, LaneState, MemoryPoolStateStore, PoolStateStore,
ProbeSlashData, StreamSlashData, Voucher, bind_node_id_domain, binding_signing_hash,
min_payment, signed_to_wire_voucher, slash_judge_domain, voucher_domain,
};
use decdn_node::buyer_pool::{OpenReported, PoolOpenPending, PoolOpener};
use decdn_node::dht::routing::{NodeId as DhtNodeId, RoutingTable};
use decdn_node::dht::{
ConfigStakerSet, NegativeProbeCache, NodeAddressResolver, OriginDirectory, PositiveProbeCache,
ProbedProvider, StakerSet, StaticNodeAddressDirectory, StaticOriginDirectory,
};
use decdn_node::metrics::Metrics;
use decdn_node::node_origin::{NodeOrigin, NodeOriginConfig, NodeOriginDeps};
use decdn_node::selection::{MAX_PROVIDER_ATTEMPTS, POOL_OPEN_CALLER_BUDGET, outer_pull_deadline};
use decdn_protocol::client::{
ChunkData, ClientBinding, ClientMessage, StreamError, StreamRequest, StreamRequestExt,
StreamResponse, StreamResponseBody, StreamResponseExt, VoucherRejectReason,
encode_stream_response,
};
use decdn_protocol::message::{
ProbeResponse, ProbeResponseBody, ProbeResponseExt, encode_probe_response,
};
use decdn_protocol::{
ALPN_CLIENT, ALPN_PROBE, CHUNK_BYTES, ContentHash, Coverage, MB_BYTES, ProbeMessage,
decode_message, encode_message, encode_stream_request, read_frame, write_frame,
};
use decdn_reputation::{LocalReputation, LocalReputationConfig};
use iroh::EndpointAddr;
use iroh::endpoint::Connection;
mod support;
use support::{
BufferedClientReader, HandlerDomains, build_handler_full, build_handler_full_configured,
cache_with_blob, empty_cache, fresh_key, local_endpoint, permissive_limiter, shutdown,
spawn_server,
};
const WIRE_FRAME: usize = 1024;
const CHAIN_ID: u64 = 421_614;
const DEPOSIT_MICRO_USDC: u64 = 10_000_000;
const PAYLOAD_LEN: usize = 1_572_864;
const RATE: u64 = 10;
const STALL_RATE: u64 = RATE / 2;
const RANK_LAST_RATE: u64 = 1_000;
const fn slash_verifying() -> Address {
Address::repeat_byte(0x11)
}
fn slash_domain() -> Eip712Domain {
slash_judge_domain(CHAIN_ID, slash_verifying())
}
fn voucher_dom() -> Eip712Domain {
voucher_domain(CHAIN_ID, Address::repeat_byte(0x34))
}
fn binding_dom() -> Eip712Domain {
bind_node_id_domain(CHAIN_ID, Address::repeat_byte(0x99))
}
type ProgressEntry = (Address, U256, U256);
#[derive(Debug)]
struct StubPoolView {
status: HashMap<B256, decdn_node::pool_view::PoolStatus>,
}
#[async_trait]
impl decdn_node::pool_view::PoolView for StubPoolView {
async fn status(&self, pool_id: B256) -> Option<decdn_node::pool_view::PoolStatus> {
self.status.get(&pool_id).copied()
}
}
#[derive(Debug)]
struct StubOpener {
pool_id: B256,
deposit: U256,
signer: Arc<PrivateKeySigner>,
voucher_domain: Eip712Domain,
recorded: Arc<Mutex<Vec<ProgressEntry>>>,
}
#[async_trait]
impl PoolOpener for StubOpener {
async fn open_or_reuse_pool(
&self,
provider_addr: Address,
_budget: Duration,
) -> Result<PoolContext> {
let recorded = self
.recorded
.lock()
.map_err(|_| anyhow::anyhow!("recorded lock poisoned"))?;
let (prior_bytes_delivered, prior_amount) = recorded
.iter()
.rev()
.find(|(provider, ..)| *provider == provider_addr)
.map_or((U256::ZERO, U256::ZERO), |(_, b, a)| (*b, *a));
Ok(PoolContext {
pool_id: self.pool_id,
provider: provider_addr,
deposit: self.deposit,
client_signer: Arc::clone(&self.signer),
voucher_domain: self.voucher_domain.clone(),
prior_bytes_delivered,
prior_amount,
client_binding: None,
capability: None,
})
}
fn record_progress(
&self,
provider_addr: Address,
pool_id: B256,
write: decdn_client::buyer_pool::ProgressWrite,
) -> Result<()> {
let totals = write.totals();
let (bytes_delivered, amount) = (totals.last_bytes, totals.last_amount);
anyhow::ensure!(
pool_id == self.pool_id,
"record_progress pool_id {pool_id} != opened pool {}",
self.pool_id
);
self.recorded
.lock()
.map_err(|_| anyhow::anyhow!("recorded lock poisoned"))?
.push((provider_addr, bytes_delivered, amount));
Ok(())
}
}
#[derive(Debug)]
struct FailingRecordOpener {
pool_id: B256,
deposit: U256,
signer: Arc<PrivateKeySigner>,
voucher_domain: Eip712Domain,
}
#[derive(Debug)]
struct WedgedOpener {
wedged: HashSet<Address>,
pool_id: B256,
deposit: U256,
signer: Arc<PrivateKeySigner>,
voucher_domain: Eip712Domain,
attempted: Arc<Mutex<Vec<Address>>>,
}
#[async_trait]
impl PoolOpener for WedgedOpener {
async fn open_or_reuse_pool(
&self,
provider_addr: Address,
budget: Duration,
) -> Result<PoolContext> {
if let Ok(mut attempted) = self.attempted.lock() {
attempted.push(provider_addr);
}
if self.wedged.contains(&provider_addr) {
tokio::time::sleep(budget).await;
return Err(anyhow::Error::new(PoolOpenPending { waited: budget }));
}
Ok(PoolContext {
pool_id: self.pool_id,
provider: provider_addr,
deposit: self.deposit,
client_signer: Arc::clone(&self.signer),
voucher_domain: self.voucher_domain.clone(),
prior_bytes_delivered: U256::ZERO,
prior_amount: U256::ZERO,
client_binding: None,
capability: None,
})
}
fn record_progress(
&self,
_provider_addr: Address,
_pool_id: B256,
_write: decdn_client::buyer_pool::ProgressWrite,
) -> Result<()> {
Ok(())
}
}
#[async_trait]
impl PoolOpener for FailingRecordOpener {
async fn open_or_reuse_pool(
&self,
provider_addr: Address,
_budget: Duration,
) -> Result<PoolContext> {
Ok(PoolContext {
pool_id: self.pool_id,
provider: provider_addr,
deposit: self.deposit,
client_signer: Arc::clone(&self.signer),
voucher_domain: self.voucher_domain.clone(),
prior_bytes_delivered: U256::ZERO,
prior_amount: U256::ZERO,
client_binding: None,
capability: None,
})
}
fn record_progress(
&self,
_provider_addr: Address,
_pool_id: B256,
_write: decdn_client::buyer_pool::ProgressWrite,
) -> Result<()> {
anyhow::bail!("simulated buyer-pool store write failure")
}
}
async fn answer_probe(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
rate: u64,
total_bytes: u64,
) -> Result<()> {
let coverage = Coverage::full(decdn_protocol::num_blocks(total_bytes));
answer_probe_with_coverage(conn, eth, slash, rate, total_bytes, coverage).await
}
async fn answer_probe_with_coverage(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
rate: u64,
total_bytes: u64,
coverage: Coverage,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read probe request: {e}"))?;
let (msg, _) = decode_message::<ProbeMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode probe request: {e}"))?;
let ProbeMessage::Request(req) = msg else {
anyhow::bail!("expected a ProbeMessage::Request");
};
let body = ProbeResponseBody {
hash: req.hash,
has_blob: true,
rate_per_mb: rate,
timestamp_us: req.timestamp_us,
};
let slash_sig = ProbeSlashData {
hash: B256::from(req.hash),
has_blob: true,
rate_per_mb: rate,
timestamp_us: req.timestamp_us,
}
.sign(eth.as_ref(), slash)
.map_err(|e| anyhow::anyhow!("sign probe slash: {e}"))?
.as_bytes()
.to_vec();
let resp = ProbeResponse { body, slash_sig };
let payload = encode_probe_response(
&resp,
Some(&ProbeResponseExt {
total_bytes: Some(total_bytes),
coverage,
}),
)
.map_err(|e| anyhow::anyhow!("encode probe response: {e}"))?;
write_frame(&mut send, &payload)
.await
.map_err(|e| anyhow::anyhow!("write probe response: {e}"))?;
let _ = send.finish();
conn.closed().await;
Ok(())
}
fn spawn_a_server(
ep: iroh::Endpoint,
client_handler: Arc<decdn_node::handlers::client::ClientHandler>,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
total_bytes: u64,
rate: u64,
) -> tokio::task::JoinHandle<()> {
use iroh::protocol::ProtocolHandler;
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
if conn.alpn() == ALPN_PROBE {
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
let handler = Arc::clone(&client_handler);
tokio::spawn(async move {
let _ = decdn_node::handlers::client::ClientProtocol::new(handler)
.accept(conn)
.await;
});
}
}
})
}
fn spawn_a_probe_counting_server(
ep: iroh::Endpoint,
client_handler: Arc<decdn_node::handlers::client::ClientHandler>,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
total_bytes: u64,
rate: u64,
probes: Arc<AtomicUsize>,
) -> tokio::task::JoinHandle<()> {
use iroh::protocol::ProtocolHandler;
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
if conn.alpn() == ALPN_PROBE {
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let probes = Arc::clone(&probes);
tokio::spawn(async move {
probes.fetch_add(1, Ordering::SeqCst);
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
let handler = Arc::clone(&client_handler);
tokio::spawn(async move {
let _ = decdn_node::handlers::client::ClientProtocol::new(handler)
.accept(conn)
.await;
});
}
}
})
}
#[allow(clippy::too_many_arguments)]
async fn provisioned_origin(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
hash: Hash,
pool_id: B256,
buyer_signer: &Arc<PrivateKeySigner>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
providers: Vec<DhtNodeId>,
addr_map: HashMap<DhtNodeId, Address>,
) -> (
NodeOrigin,
CacheEngine,
Arc<Mutex<Vec<ProgressEntry>>>,
tempfile::TempDir,
) {
provisioned_origin_with_deadlines(
ep_b,
b_dht,
hash,
pool_id,
buyer_signer,
local_rep,
metrics,
providers,
addr_map,
DEFAULT_TEST_PULL_DEADLINES,
0,
)
.await
}
const DEFAULT_TEST_PULL_DEADLINES: (Duration, Duration) =
(Duration::from_mins(1), Duration::from_mins(1));
#[allow(clippy::too_many_arguments)]
async fn provisioned_origin_with_deadlines(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
hash: Hash,
pool_id: B256,
buyer_signer: &Arc<PrivateKeySigner>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
providers: Vec<DhtNodeId>,
addr_map: HashMap<DhtNodeId, Address>,
(pull_timeout, stall_timeout): (Duration, Duration),
max_blob_size_bytes: u64,
) -> (
NodeOrigin,
CacheEngine,
Arc<Mutex<Vec<ProgressEntry>>>,
tempfile::TempDir,
) {
let (buyer, recorded) = stub_opener(pool_id, buyer_signer);
let (origin, engine, engine_tmp) = build_origin_with_timeout(
ep_b,
b_dht,
hash,
buyer,
local_rep,
metrics,
providers,
addr_map,
pull_timeout,
stall_timeout,
max_blob_size_bytes,
)
.await;
(origin, engine, recorded, engine_tmp)
}
fn stub_opener(
pool_id: B256,
signer: &Arc<PrivateKeySigner>,
) -> (Arc<dyn PoolOpener>, Arc<Mutex<Vec<ProgressEntry>>>) {
let recorded: Arc<Mutex<Vec<ProgressEntry>>> = Arc::new(Mutex::new(Vec::new()));
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(signer),
voucher_domain: voucher_dom(),
recorded: Arc::clone(&recorded),
}) as Arc<dyn PoolOpener>;
(buyer, recorded)
}
#[allow(clippy::too_many_arguments)]
async fn provisioned_origin_with_ceiling(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
hash: Hash,
pool_id: B256,
buyer_signer: &Arc<PrivateKeySigner>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
providers: Vec<DhtNodeId>,
addr_map: HashMap<DhtNodeId, Address>,
max_blob_size_bytes: u64,
) -> (NodeOrigin, CacheEngine, tempfile::TempDir) {
let recorded: Arc<Mutex<Vec<ProgressEntry>>> = Arc::new(Mutex::new(Vec::new()));
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(buyer_signer),
voucher_domain: voucher_dom(),
recorded,
}) as Arc<dyn PoolOpener>;
build_origin_with_timeout(
ep_b,
b_dht,
hash,
buyer,
local_rep,
metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
max_blob_size_bytes,
)
.await
}
#[allow(clippy::too_many_arguments, clippy::expect_used)]
async fn build_origin(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
hash: Hash,
buyer: Arc<dyn PoolOpener>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
providers: Vec<DhtNodeId>,
addr_map: HashMap<DhtNodeId, Address>,
) -> (NodeOrigin, CacheEngine, tempfile::TempDir) {
build_origin_with_timeout(
ep_b,
b_dht,
hash,
buyer,
local_rep,
metrics,
providers,
addr_map,
DEFAULT_TEST_PULL_DEADLINES.0,
DEFAULT_TEST_PULL_DEADLINES.1,
0,
)
.await
}
#[allow(clippy::too_many_arguments, clippy::expect_used)]
async fn build_origin_with_timeout(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
hash: Hash,
buyer: Arc<dyn PoolOpener>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
providers: Vec<DhtNodeId>,
addr_map: HashMap<DhtNodeId, Address>,
pull_timeout: Duration,
stall_timeout: Duration,
max_blob_size_bytes: u64,
) -> (NodeOrigin, CacheEngine, tempfile::TempDir) {
build_origin_with_negative_cache(
ep_b,
b_dht,
hash,
buyer,
local_rep,
metrics,
providers,
addr_map,
pull_timeout,
stall_timeout,
max_blob_size_bytes,
NegativeProbeCache::new(),
)
.await
}
#[allow(clippy::too_many_arguments, clippy::expect_used)]
async fn build_origin_with_negative_cache(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
hash: Hash,
buyer: Arc<dyn PoolOpener>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
providers: Vec<DhtNodeId>,
addr_map: HashMap<DhtNodeId, Address>,
pull_timeout: Duration,
stall_timeout: Duration,
max_blob_size_bytes: u64,
negative_cache: NegativeProbeCache,
) -> (NodeOrigin, CacheEngine, tempfile::TempDir) {
build_origin_with_probe_caches(
ep_b,
b_dht,
hash,
buyer,
local_rep,
metrics,
providers,
addr_map,
pull_timeout,
stall_timeout,
max_blob_size_bytes,
negative_cache,
PositiveProbeCache::new(),
U256::ZERO,
)
.await
}
#[allow(clippy::too_many_arguments, clippy::expect_used)]
async fn build_origin_with_probe_caches(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
_hash: Hash,
buyer: Arc<dyn PoolOpener>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
providers: Vec<DhtNodeId>,
addr_map: HashMap<DhtNodeId, Address>,
pull_timeout: Duration,
stall_timeout: Duration,
max_blob_size_bytes: u64,
negative_cache: NegativeProbeCache,
probe_cache: PositiveProbeCache,
directory_namespace: U256,
) -> (NodeOrigin, CacheEngine, tempfile::TempDir) {
build_origin_with_discovery(
ep_b,
b_dht,
buyer,
local_rep,
metrics,
FixtureDiscovery::directory_only(providers, directory_namespace),
addr_map,
pull_timeout,
stall_timeout,
max_blob_size_bytes,
negative_cache,
probe_cache,
)
.await
}
struct FixtureDiscovery {
stakers: HashSet<DhtNodeId>,
directory: HashMap<U256, Vec<DhtNodeId>>,
dht_peers: Vec<(iroh::PublicKey, std::net::SocketAddr)>,
probe_fanout: usize,
}
impl FixtureDiscovery {
fn directory_only(providers: Vec<DhtNodeId>, namespace: U256) -> Self {
Self {
stakers: providers.iter().copied().collect(),
directory: HashMap::from([(namespace, providers)]),
dht_peers: Vec::new(),
probe_fanout: 5,
}
}
}
#[allow(clippy::too_many_arguments, clippy::expect_used)]
async fn build_origin_with_discovery(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
buyer: Arc<dyn PoolOpener>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
discovery: FixtureDiscovery,
addr_map: HashMap<DhtNodeId, Address>,
pull_timeout: Duration,
stall_timeout: Duration,
max_blob_size_bytes: u64,
negative_cache: NegativeProbeCache,
probe_cache: PositiveProbeCache,
) -> (NodeOrigin, CacheEngine, tempfile::TempDir) {
let FixtureDiscovery {
stakers,
directory,
dht_peers,
probe_fanout,
} = discovery;
let mut routing = RoutingTable::new(b_dht);
for (peer_id, peer_addr) in dht_peers {
let peer = DhtNodeId::from_bytes(*peer_id.as_bytes());
decdn_node::dht::client::find_node(
ep_b,
EndpointAddr::new(peer_id).with_ip_addr(peer_addr),
peer,
b_dht,
)
.await
.expect("prime the fixture's address cache for a DHT peer");
routing.insert(peer);
}
let (engine, engine_tmp) = throwaway_engine()
.await
.expect("throwaway engine for the node-origin fixture");
let origin = NodeOrigin::new();
origin.provision(NodeOriginDeps {
endpoint: ep_b.clone(),
dial_runtime: tokio::runtime::Handle::current(),
routing_table: Arc::new(Mutex::new(routing)),
staker_set: Arc::new(ConfigStakerSet::new(stakers)) as Arc<dyn StakerSet>,
origin_directory: Arc::new(StaticOriginDirectory::new(directory))
as Arc<dyn OriginDirectory>,
addr_resolver: Arc::new(StaticNodeAddressDirectory::new(addr_map))
as Arc<dyn NodeAddressResolver>,
buyer,
self_id: b_dht,
slash_domain: slash_domain(),
bind_domain: binding_dom(),
local_rep: Arc::clone(local_rep),
negative_cache,
probe_cache,
metrics: Arc::clone(metrics),
registry_regions: Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())),
config: NodeOriginConfig {
probe_fanout,
pull_timeout,
stall_window: stall_timeout,
min_throughput_bps: 0,
max_blob_size_bytes,
max_rate_per_mb: 0,
event_poll_interval: Duration::from_millis(50),
lookup: decdn_node::dht::LookupConfig::default(),
own_region: None,
serve_economics: std::sync::Arc::new(decdn_node::serve_economics::OffPolicy),
operator_shares: decdn_node::fee_shares::OperatorShares::new(6000),
frequency_estimator: None,
sell_rate_base: 0,
warming: std::sync::Arc::new(decdn_node::warming_allowance::WarmingAllowance::new(
1_000_000_000,
0,
)),
},
ledgers: Arc::new(decdn_node::buyer_ledgers::BuyerLedgers::default()),
wedged_providers: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
engine: engine.clone(),
});
(origin, engine, engine_tmp)
}
#[allow(clippy::too_many_arguments, clippy::expect_used)]
async fn build_origin_seeded_ranking(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
hash: Hash,
buyer: Arc<dyn PoolOpener>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
ranked: &[(DhtNodeId, u64)],
addr_map: HashMap<DhtNodeId, Address>,
pull_timeout: Duration,
stall_timeout: Duration,
) -> (NodeOrigin, CacheEngine, tempfile::TempDir) {
let probe_cache = PositiveProbeCache::new();
probe_cache.insert(
ContentHash::from_bytes(*hash.as_bytes()),
ranked
.iter()
.map(|&(node_id, rate_per_mb)| ProbedProvider {
node_id,
rate_per_mb,
rtt_ms: 1,
coverage: decdn_protocol::Coverage::full(1),
total_bytes_hint: None,
})
.collect(),
);
let providers = ranked.iter().map(|&(id, _)| id).collect();
build_origin_with_probe_caches(
ep_b,
b_dht,
hash,
buyer,
local_rep,
metrics,
providers,
addr_map,
pull_timeout,
stall_timeout,
0,
NegativeProbeCache::new(),
probe_cache,
U256::ZERO,
)
.await
}
#[allow(clippy::too_many_arguments, clippy::expect_used)]
async fn build_origin_multi_hash(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
hashes: &[Hash],
buyer: Arc<dyn PoolOpener>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
providers: &[DhtNodeId],
addr_map: HashMap<DhtNodeId, Address>,
pull_timeout: Duration,
stall_timeout: Duration,
) -> (NodeOrigin, CacheEngine, tempfile::TempDir) {
let mut dir = HashMap::new();
for _h in hashes {
dir.insert(U256::ZERO, providers.to_vec());
}
let (engine, engine_tmp) = throwaway_engine()
.await
.expect("throwaway engine for the node-origin fixture");
let origin = NodeOrigin::new();
origin.provision(NodeOriginDeps {
endpoint: ep_b.clone(),
dial_runtime: tokio::runtime::Handle::current(),
routing_table: Arc::new(Mutex::new(RoutingTable::new(b_dht))),
staker_set: Arc::new(ConfigStakerSet::new(providers.iter().copied().collect()))
as Arc<dyn StakerSet>,
origin_directory: Arc::new(StaticOriginDirectory::new(dir)) as Arc<dyn OriginDirectory>,
addr_resolver: Arc::new(StaticNodeAddressDirectory::new(addr_map))
as Arc<dyn NodeAddressResolver>,
buyer,
self_id: b_dht,
slash_domain: slash_domain(),
bind_domain: binding_dom(),
local_rep: Arc::clone(local_rep),
negative_cache: NegativeProbeCache::new(),
probe_cache: PositiveProbeCache::new(),
metrics: Arc::clone(metrics),
registry_regions: Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())),
config: NodeOriginConfig {
probe_fanout: 5,
pull_timeout,
stall_window: stall_timeout,
min_throughput_bps: 0,
max_blob_size_bytes: 0,
max_rate_per_mb: 0,
event_poll_interval: Duration::from_millis(50),
lookup: decdn_node::dht::LookupConfig::default(),
own_region: None,
serve_economics: std::sync::Arc::new(decdn_node::serve_economics::OffPolicy),
operator_shares: decdn_node::fee_shares::OperatorShares::new(6000),
frequency_estimator: None,
sell_rate_base: 0,
warming: std::sync::Arc::new(decdn_node::warming_allowance::WarmingAllowance::new(
1_000_000_000,
0,
)),
},
ledgers: Arc::new(decdn_node::buyer_ledgers::BuyerLedgers::default()),
wedged_providers: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
engine: engine.clone(),
});
(origin, engine, engine_tmp)
}
fn one_provider(
a_dht: DhtNodeId,
a_eth_addr: Address,
) -> (Vec<DhtNodeId>, HashMap<DhtNodeId, Address>) {
let mut addr_map = HashMap::new();
addr_map.insert(a_dht, a_eth_addr);
(vec![a_dht], addr_map)
}
fn progress_log(recorded: &Arc<Mutex<Vec<ProgressEntry>>>) -> Result<Vec<ProgressEntry>> {
let log = recorded
.lock()
.map_err(|_| anyhow::anyhow!("recorded lock poisoned"))?;
Ok(log.clone())
}
fn assert_counter(metrics: &Arc<Metrics>, name: &str, value: u64) -> Result<()> {
let text = metrics
.encode()
.map_err(|e| anyhow::anyhow!("encode metrics: {e}"))?;
let want = format!("decdn_{name} {value}");
anyhow::ensure!(
text.lines().any(|l| l == want),
"expected metric line `{want}`; got:\n{text}"
);
Ok(())
}
async fn assert_relay_counted(metrics: &Arc<Metrics>, blob_len: u64, pull_legs: u64) -> Result<()> {
let outbound_completed = "streams_completed_total{direction=\"outbound\"}";
let deadline = std::time::Instant::now() + Duration::from_secs(10);
while counter_value(metrics, outbound_completed)? == 0 && std::time::Instant::now() < deadline {
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_counter(metrics, outbound_completed, 1)?;
assert_counter(metrics, "streams_failed_total{direction=\"outbound\"}", 0)?;
anyhow::ensure!(
counter_value(metrics, "bytes_received_total")? >= blob_len,
"the relay must count the blob it pulled"
);
anyhow::ensure!(
counter_value(metrics, "bytes_served_total")? >= blob_len,
"the relay must count the blob it served"
);
assert_counter(metrics, "node_pull_first_byte_seconds_count", pull_legs)?;
assert_counter(metrics, "serve_first_byte_miss_seconds_count", 1)?;
Ok(())
}
fn counter_value(metrics: &Arc<Metrics>, name: &str) -> Result<u64> {
let text = metrics
.encode()
.map_err(|e| anyhow::anyhow!("encode metrics: {e}"))?;
let prefix = format!("decdn_{name} ");
let val = text
.lines()
.find_map(|l| l.strip_prefix(&prefix))
.and_then(|v| v.trim().parse::<u64>().ok())
.unwrap_or(0);
Ok(val)
}
async fn throwaway_engine() -> Result<(CacheEngine, tempfile::TempDir)> {
let dir = tempfile::tempdir()?;
let engine = CacheEngine::open(dir.path(), vec![], 16).await?;
Ok((engine, dir))
}
async fn cache_with_blobs(payloads: &[&[u8]]) -> Result<(CacheEngine, tempfile::TempDir)> {
let origin_dir = tempfile::tempdir()?;
for payload in payloads {
let hex = Hash::new(payload).to_hex();
let shard = hex
.get(..2)
.ok_or_else(|| anyhow::anyhow!("hex too short"))?;
let dir = origin_dir.path().join(shard);
std::fs::create_dir_all(&dir)?;
std::fs::write(dir.join(hex.as_str()), payload)?;
}
let cache_dir = tempfile::tempdir()?;
let origin = Arc::new(FilesystemOrigin::new(origin_dir.path()).await?);
let cache = CacheEngine::open(cache_dir.path(), vec![origin as Arc<dyn Origin>], 16).await?;
for payload in payloads {
let _ = cache.get(Hash::new(payload)).await?; }
drop(origin_dir);
Ok((cache, cache_dir))
}
async fn cache_with_fs_origin_only(
payload: &[u8],
) -> Result<(
CacheEngine,
Hash,
Arc<CacheMetrics>,
tempfile::TempDir,
tempfile::TempDir,
)> {
let hash = Hash::new(payload);
let origin_dir = tempfile::tempdir()?;
let hex = hash.to_hex();
let shard = hex
.get(..2)
.ok_or_else(|| anyhow::anyhow!("hex too short"))?;
let dir = origin_dir.path().join(shard);
std::fs::create_dir_all(&dir)?;
std::fs::write(dir.join(hex.as_str()), payload)?;
let cache_dir = tempfile::tempdir()?;
let origin = Arc::new(FilesystemOrigin::new(origin_dir.path()).await?);
let metrics = Arc::new(CacheMetrics::default());
let cache = CacheEngine::open_full(
cache_dir.path(),
vec![origin as Arc<dyn Origin>],
16,
PinnedHashes::empty(),
RetryPolicy::default(),
CircuitBreakerPolicy::default(),
Some(Arc::clone(&metrics)),
Duration::ZERO,
)
.await?;
anyhow::ensure!(!cache.has(hash).await?, "upstream store must start empty");
Ok((cache, hash, metrics, origin_dir, cache_dir))
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn node_origin_pull_chains_reactive_origin_via_client_binding() -> Result<()> {
let payload = vec![0x9Eu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, cache_a_metrics, _origin_tmp_a, _cache_tmp_a) =
cache_with_fs_origin_only(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xB1);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_a = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics_a);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full_configured(
a_id,
&a_eth,
&metrics_a,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
|deps| deps.local_populate = Some(Duration::from_secs(20)),
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, engine, _recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
pool_id,
&b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
)
.await;
let fetched = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("node-origin chained fetch failed: {e}"))?;
anyhow::ensure!(
matches!(fetched, OriginFetch::AlreadyAdmitted),
"node-origin did not admit the blob; the chained reactive pull did not fire"
);
let bytes = engine.get(hash).await?;
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"chained-pull bytes mismatch"
);
anyhow::ensure!(
cache_a_metrics.origin_fetches.get() == 1,
"upstream A must reactively pull its origin exactly once, got {}",
cache_a_metrics.origin_fetches.get()
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn node_origin_pull_fills_and_records_reputation() -> Result<()> {
let spans = support::capture_spans();
let payload = vec![0xABu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xA1);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, engine, recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
pool_id,
&b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
)
.await;
let fetched = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("node-origin fetch failed: {e}"))?;
anyhow::ensure!(
matches!(fetched, OriginFetch::AlreadyAdmitted),
"node-origin did not admit the blob; expected the pull to deliver it"
);
let bytes = engine.get(hash).await?;
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"pulled bytes mismatch"
);
anyhow::ensure!(
local_rep.score(a_id) > 0.5,
"local score should rise after a clean delivery, got {}",
local_rep.score(a_id)
);
assert_counter(&b_metrics, "node_pull_attempts_total", 1)?;
assert_counter(&b_metrics, "node_pull_success_total", 1)?;
assert_counter(&b_metrics, "probe_collection_latency_seconds_count", 1)?;
assert_counter(&b_metrics, "node_pull_first_byte_seconds_count", 1)?;
let expected_wire =
decdn_cache::range_pull::bao_encoded_size(total_bytes, &bao_tree::ChunkRanges::all());
let expected_amount = U256::from(expected_wire)
.saturating_mul(U256::from(RATE))
.div_ceil(U256::from(MB_BYTES));
anyhow::ensure!(
progress_log(&recorded)?
== vec![(a_eth.address(), U256::from(expected_wire), expected_amount)],
"expected one persisted progress entry with the final voucher totals, got {:?}",
progress_log(&recorded)?
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
let key = hash.to_string();
let pulls = spans.matching("upstream_stream", "hash", &key);
anyhow::ensure!(
pulls.iter().any(|s| s.parent == Some("node_pull")
&& s.fields.get("outcome").map(String::as_str) == Some("filled")),
"upstream_stream: {pulls:?}"
);
let opens = spans.matching("open_progressive_pull", "hash", &key);
anyhow::ensure!(
!opens.is_empty()
&& opens.iter().all(|s| s.parent == Some("upstream_stream")
&& s.fields.contains_key("byte_offset")
&& s.fields.contains_key("byte_len")),
"open_progressive_pull must nest under upstream_stream with its range: {opens:?}"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn large_blob_populates_via_streaming_pull() -> Result<()> {
const LARGE_PAYLOAD_LEN: usize = 8 * 1024 * 1024;
let mut payload = vec![0u8; LARGE_PAYLOAD_LEN];
let mut x: u32 = 0x9e37_79b9;
for b in &mut payload {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
*b = x.to_le_bytes().first().copied().unwrap_or(0);
}
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(LARGE_PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xA2);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let origin = NodeOrigin::new();
let cache_dir_b = tempfile::tempdir()?;
let engine_b = CacheEngine::open(
cache_dir_b.path(),
vec![Arc::new(origin.clone()) as Arc<dyn Origin>],
16,
)
.await?;
let recorded: Arc<Mutex<Vec<ProgressEntry>>> = Arc::new(Mutex::new(Vec::new()));
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::clone(&recorded),
}) as Arc<dyn PoolOpener>;
let mut dir = HashMap::new();
dir.insert(U256::ZERO, providers.clone());
origin.provision(NodeOriginDeps {
endpoint: ep_b.clone(),
dial_runtime: tokio::runtime::Handle::current(),
routing_table: Arc::new(Mutex::new(RoutingTable::new(DhtNodeId::from_bytes(
*b_id.as_bytes(),
)))),
staker_set: Arc::new(ConfigStakerSet::new(providers.into_iter().collect()))
as Arc<dyn StakerSet>,
origin_directory: Arc::new(StaticOriginDirectory::new(dir)) as Arc<dyn OriginDirectory>,
addr_resolver: Arc::new(StaticNodeAddressDirectory::new(addr_map))
as Arc<dyn NodeAddressResolver>,
buyer,
self_id: DhtNodeId::from_bytes(*b_id.as_bytes()),
slash_domain: slash_domain(),
bind_domain: binding_dom(),
local_rep: Arc::clone(&local_rep),
negative_cache: NegativeProbeCache::new(),
probe_cache: PositiveProbeCache::new(),
metrics: Arc::clone(&b_metrics),
registry_regions: Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())),
config: NodeOriginConfig {
probe_fanout: 5,
pull_timeout: DEFAULT_TEST_PULL_DEADLINES.0,
stall_window: DEFAULT_TEST_PULL_DEADLINES.1,
min_throughput_bps: 0,
max_blob_size_bytes: 0,
max_rate_per_mb: 0,
event_poll_interval: Duration::from_millis(50),
lookup: decdn_node::dht::LookupConfig::default(),
own_region: None,
serve_economics: std::sync::Arc::new(decdn_node::serve_economics::OffPolicy),
operator_shares: decdn_node::fee_shares::OperatorShares::new(6000),
frequency_estimator: None,
sell_rate_base: 0,
warming: std::sync::Arc::new(decdn_node::warming_allowance::WarmingAllowance::new(
1_000_000_000,
0,
)),
},
ledgers: Arc::new(decdn_node::buyer_ledgers::BuyerLedgers::default()),
wedged_providers: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
engine: engine_b.clone(),
});
engine_b.populate(hash).await?;
anyhow::ensure!(
engine_b.has(hash).await?,
"blob must be present after populate"
);
let got = engine_b.get(hash).await?;
anyhow::ensure!(got.len() as u64 == total_bytes, "full blob length");
anyhow::ensure!(got.as_ref() == payload.as_slice(), "content matches");
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
async fn serve_wrong_bytes(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
served: &[u8],
rate: u64,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read request: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode request: {e}"))?
.0
};
let ClientMessage::StreamRequest(req) = req_msg else {
anyhow::bail!("lying upstream: expected a StreamRequest");
};
let body = StreamResponseBody {
hash: req.hash,
ok: true,
rate_per_mb: rate,
total_bytes: u64::try_from(served.len()).unwrap_or(u64::MAX),
pool_id: req.pool_id,
timestamp_us: req.timestamp_us,
};
let slash_sig = StreamSlashData::from_response_body(&body)
.sign(eth.as_ref(), slash)
.map_err(|e| anyhow::anyhow!("slash sign: {e}"))?
.as_bytes()
.to_vec();
let resp = StreamResponse { body, slash_sig };
write_frame(
&mut send,
&encode_stream_response(&resp, Some(&StreamResponseExt { error: None }))?,
)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
for chunk in served.chunks(WIRE_FRAME) {
write_frame(
&mut send,
&encode_message(&ClientMessage::ChunkData(ChunkData::new(chunk.to_vec())?))?,
)
.await
.map_err(|e| anyhow::anyhow!("write chunk: {e}"))?;
}
let voucher_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read voucher: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode voucher: {e}"))?
.0
};
let ClientMessage::Voucher(_) = voucher_msg else {
anyhow::bail!("expected a closing Voucher, got {voucher_msg:?}");
};
write_frame(&mut send, &encode_message(&ClientMessage::StreamEnd)?)
.await
.map_err(|e| anyhow::anyhow!("write end: {e}"))?;
let _ = send.finish();
conn.closed().await;
Ok(())
}
fn spawn_a_lying_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
served: Vec<u8>,
advertised_bytes: u64,
rate: u64,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let served = served.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, advertised_bytes).await;
});
} else {
tokio::spawn(async move {
let _ = serve_wrong_bytes(conn, ð, &dom, &served, rate).await;
});
}
}
})
}
async fn serve_gated_correct_bytes(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
served: &[u8],
rate: u64,
received: &tokio::sync::Notify,
release: &tokio::sync::Notify,
) -> Result<()> {
loop {
let Ok((mut send, mut recv)) = conn.accept_bi().await else {
return Ok(());
};
let req_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read request: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode request: {e}"))?
.0
};
let ClientMessage::StreamRequest(req) = req_msg else {
anyhow::bail!("gated upstream: expected a StreamRequest");
};
let body = StreamResponseBody {
hash: req.hash,
ok: true,
rate_per_mb: rate,
total_bytes: u64::try_from(served.len()).unwrap_or(u64::MAX),
pool_id: req.pool_id,
timestamp_us: req.timestamp_us,
};
let slash_sig = StreamSlashData::from_response_body(&body)
.sign(eth.as_ref(), slash)
.map_err(|e| anyhow::anyhow!("slash sign: {e}"))?
.as_bytes()
.to_vec();
let resp = StreamResponse { body, slash_sig };
let resp_ext = StreamResponseExt { error: None };
write_frame(&mut send, &encode_stream_response(&resp, Some(&resp_ext))?)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
if req.byte_offset == 0 && req.byte_len == 0 {
let _ = send.finish();
continue;
}
received.notify_one();
release.notified().await;
for chunk in served.chunks(WIRE_FRAME) {
write_frame(
&mut send,
&encode_message(&ClientMessage::ChunkData(ChunkData::new(chunk.to_vec())?))?,
)
.await
.map_err(|e| anyhow::anyhow!("write chunk: {e}"))?;
}
let voucher_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read voucher: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode voucher: {e}"))?
.0
};
let ClientMessage::Voucher(_) = voucher_msg else {
anyhow::bail!("gated upstream: expected a closing Voucher, got {voucher_msg:?}");
};
write_frame(&mut send, &encode_message(&ClientMessage::StreamEnd)?)
.await
.map_err(|e| anyhow::anyhow!("write end: {e}"))?;
let _ = send.finish();
conn.closed().await;
return Ok(());
}
}
#[allow(clippy::too_many_arguments)]
fn spawn_a_gated_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
served: Vec<u8>,
advertised_bytes: u64,
rate: u64,
received: Arc<tokio::sync::Notify>,
release: Arc<tokio::sync::Notify>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let served = served.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, advertised_bytes).await;
});
} else {
let received = Arc::clone(&received);
let release = Arc::clone(&release);
tokio::spawn(async move {
let _ = serve_gated_correct_bytes(
conn, ð, &dom, &served, rate, &received, &release,
)
.await;
});
}
}
})
}
async fn spawn_gated_node_a(
payload: &[u8],
) -> Result<(
iroh::PublicKey,
std::net::SocketAddr,
Arc<PrivateKeySigner>,
iroh::Endpoint,
tokio::task::JoinHandle<()>,
Arc<tokio::sync::Notify>,
Arc<tokio::sync::Notify>,
)> {
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let received = Arc::new(tokio::sync::Notify::new());
let release = Arc::new(tokio::sync::Notify::new());
let advertised_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let task_a = spawn_a_gated_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
payload.to_vec(),
advertised_bytes,
RATE,
Arc::clone(&received),
Arc::clone(&release),
);
Ok((a_id, addr_a, a_eth, ep_a, task_a, received, release))
}
fn spawn_server_concurrent(
server_ep: iroh::Endpoint,
handler: Arc<decdn_node::handlers::client::ClientHandler>,
) -> tokio::task::JoinHandle<()> {
use decdn_node::handlers::client::ClientProtocol;
use iroh::protocol::ProtocolHandler;
tokio::spawn(async move {
while let Some(incoming) = server_ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let handler = Arc::clone(&handler);
tokio::spawn(async move {
let _ = ClientProtocol::new(handler).accept(conn).await;
});
}
})
}
async fn serve_then_reject_voucher(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
served: &[u8],
rate: u64,
reason: VoucherRejectReason,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read request: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode request: {e}"))?
.0
};
let ClientMessage::StreamRequest(req) = req_msg else {
anyhow::bail!("voucher-rejecting upstream: expected a StreamRequest");
};
let body = StreamResponseBody {
hash: req.hash,
ok: true,
rate_per_mb: rate,
total_bytes: u64::try_from(served.len()).unwrap_or(u64::MAX),
pool_id: req.pool_id,
timestamp_us: req.timestamp_us,
};
let slash_sig = StreamSlashData::from_response_body(&body)
.sign(eth.as_ref(), slash)
.map_err(|e| anyhow::anyhow!("slash sign: {e}"))?
.as_bytes()
.to_vec();
let resp = StreamResponse { body, slash_sig };
write_frame(
&mut send,
&encode_stream_response(&resp, Some(&StreamResponseExt { error: None }))?,
)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
for chunk in served.chunks(WIRE_FRAME) {
write_frame(
&mut send,
&encode_message(&ClientMessage::ChunkData(ChunkData::new(chunk.to_vec())?))?,
)
.await
.map_err(|e| anyhow::anyhow!("write chunk: {e}"))?;
}
let voucher_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read voucher: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode voucher: {e}"))?
.0
};
let ClientMessage::Voucher(_) = voucher_msg else {
anyhow::bail!("voucher-rejecting upstream: expected a Voucher");
};
write_frame(
&mut send,
&encode_message(&ClientMessage::StreamError(StreamError::VoucherRejected {
reason,
bundle: None,
}))?,
)
.await
.map_err(|e| anyhow::anyhow!("write voucher rejection: {e}"))?;
let _ = send.finish();
conn.closed().await;
Ok(())
}
async fn serve_then_error_on_voucher(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
served: &[u8],
rate: u64,
error: StreamError,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read request: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode request: {e}"))?
.0
};
let ClientMessage::StreamRequest(req) = req_msg else {
anyhow::bail!("voucher-erroring upstream: expected a StreamRequest");
};
let body = StreamResponseBody {
hash: req.hash,
ok: true,
rate_per_mb: rate,
total_bytes: u64::try_from(served.len()).unwrap_or(u64::MAX),
pool_id: req.pool_id,
timestamp_us: req.timestamp_us,
};
let slash_sig = StreamSlashData::from_response_body(&body)
.sign(eth.as_ref(), slash)
.map_err(|e| anyhow::anyhow!("slash sign: {e}"))?
.as_bytes()
.to_vec();
let resp = StreamResponse { body, slash_sig };
write_frame(
&mut send,
&encode_stream_response(&resp, Some(&StreamResponseExt { error: None }))?,
)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
for chunk in served.chunks(WIRE_FRAME) {
write_frame(
&mut send,
&encode_message(&ClientMessage::ChunkData(ChunkData::new(chunk.to_vec())?))?,
)
.await
.map_err(|e| anyhow::anyhow!("write chunk: {e}"))?;
}
let voucher_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read voucher: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode voucher: {e}"))?
.0
};
let ClientMessage::Voucher(_) = voucher_msg else {
anyhow::bail!("voucher-erroring upstream: expected a Voucher");
};
write_frame(
&mut send,
&encode_message(&ClientMessage::StreamError(error))?,
)
.await
.map_err(|e| anyhow::anyhow!("write voucher-reply error: {e}"))?;
let _ = send.finish();
conn.closed().await;
Ok(())
}
fn spawn_a_voucher_erroring_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
served: Vec<u8>,
advertised_bytes: u64,
rate: u64,
error: StreamError,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let served = served.clone();
let error = error.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, advertised_bytes).await;
});
} else {
tokio::spawn(async move {
let _ =
serve_then_error_on_voucher(conn, ð, &dom, &served, rate, error).await;
});
}
}
})
}
fn spawn_a_voucher_rejecting_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
served: Vec<u8>,
advertised_bytes: u64,
rate: u64,
reason: VoucherRejectReason,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let served = served.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, advertised_bytes).await;
});
} else {
tokio::spawn(async move {
let _ =
serve_then_reject_voucher(conn, ð, &dom, &served, rate, reason).await;
});
}
}
})
}
#[allow(clippy::too_many_arguments)]
fn spawn_a_voucher_rejecting_server_with_counters(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
served: Vec<u8>,
advertised_bytes: u64,
rate: u64,
reason: VoucherRejectReason,
probes: Arc<AtomicUsize>,
streams: Arc<AtomicUsize>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let served = served.clone();
if conn.alpn() == ALPN_PROBE {
let probes = Arc::clone(&probes);
tokio::spawn(async move {
probes.fetch_add(1, Ordering::SeqCst);
let _ = answer_probe(conn, ð, &dom, rate, advertised_bytes).await;
});
} else {
let streams = Arc::clone(&streams);
tokio::spawn(async move {
streams.fetch_add(1, Ordering::SeqCst);
let _ =
serve_then_reject_voucher(conn, ð, &dom, &served, rate, reason).await;
});
}
}
})
}
fn spawn_a_probe_ok_client_dead_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
total_bytes: u64,
rate: u64,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
conn.close(0u32.into(), b"dead");
}
}
})
}
fn spawn_a_stalling_server(
ep: iroh::Endpoint,
s_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
total_bytes: u64,
rate: u64,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&s_eth);
let dom = slash.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
tokio::spawn(async move {
if let Ok((_send, mut recv)) = conn.accept_bi().await {
let _ = read_frame(&mut recv).await;
conn.closed().await;
}
});
}
}
})
}
fn spawn_a_counting_stalling_server(
ep: iroh::Endpoint,
s_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
total_bytes: u64,
rate: u64,
streams: Arc<AtomicUsize>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&s_eth);
let dom = slash.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
streams.fetch_add(1, Ordering::SeqCst);
tokio::spawn(async move {
if let Ok((_send, mut recv)) = conn.accept_bi().await {
let _ = read_frame(&mut recv).await;
conn.closed().await;
}
});
}
}
})
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)]
async fn node_origin_pull_falls_through_a_stalled_candidate() -> Result<()> {
let payload = vec![0xCDu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xA2);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let s_sk = fresh_key();
let s_id = s_sk.public();
let s_eth = Arc::new(PrivateKeySigner::random());
let (ep_s, addr_s) =
local_endpoint(s_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_s = spawn_a_stalling_server(
ep_s.clone(),
Arc::clone(&s_eth),
slash_domain(),
total_bytes,
STALL_RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for (id, addr) in [(a_id, addr_a), (s_id, addr_s)] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(id).with_ip_addr(addr),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let s_dht = DhtNodeId::from_bytes(*s_id.as_bytes());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let mut addr_map = HashMap::new();
addr_map.insert(s_dht, s_eth.address());
addr_map.insert(a_dht, a_eth.address());
let recorded: Arc<Mutex<Vec<ProgressEntry>>> = Arc::new(Mutex::new(Vec::new()));
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::clone(&recorded),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_seeded_ranking(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
&[(s_dht, STALL_RATE), (a_dht, RATE)],
addr_map,
Duration::from_secs(1),
Duration::from_secs(20),
)
.await;
let fetched = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("node-origin fetch failed: {e}"))?;
anyhow::ensure!(
matches!(fetched, OriginFetch::AlreadyAdmitted),
"expected the blob from the honest fallback candidate"
);
let bytes = engine.get(hash).await?;
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"pulled bytes mismatch"
);
anyhow::ensure!(
(local_rep.score(s_id) - 0.5).abs() < f64::EPSILON,
"the timed-out staller's local score must stay neutral, got {}",
local_rep.score(s_id)
);
anyhow::ensure!(
local_rep.score(a_id) > 0.5,
"honest provider should score a clean delivery, got {}",
local_rep.score(a_id)
);
assert_counter(&b_metrics, "node_pull_success_total", 1)?;
assert_counter(&b_metrics, "node_pull_timeout_total", 1)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
assert_counter(&b_metrics, "node_pull_voucher_rejected_total", 0)?;
assert_counter(&b_metrics, "node_pull_corruption_total", 0)?;
shutdown([task_a, task_s], [&ep_b, &ep_a, &ep_s]).await?;
Ok(())
}
#[allow(clippy::too_many_lines, clippy::similar_names)]
async fn wedged_open_does_not_starve_the_candidate_loop() -> Result<()> {
let payload = vec![0xE1u8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xE1);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let w_sk = fresh_key();
let w_id = w_sk.public();
let w_eth = Arc::new(PrivateKeySigner::random());
let (ep_w, addr_w) =
local_endpoint(w_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_w = spawn_a_stalling_server(
ep_w.clone(),
Arc::clone(&w_eth),
slash_domain(),
total_bytes,
STALL_RATE,
);
let w2_sk = fresh_key();
let w2_id = w2_sk.public();
let w2_eth = Arc::new(PrivateKeySigner::random());
let (ep_w2, addr_w2) =
local_endpoint(w2_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_w2 = spawn_a_stalling_server(
ep_w2.clone(),
Arc::clone(&w2_eth),
slash_domain(),
total_bytes,
STALL_RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for (id, addr) in [(a_id, addr_a), (w_id, addr_w), (w2_id, addr_w2)] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(id).with_ip_addr(addr),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let w_dht = DhtNodeId::from_bytes(*w_id.as_bytes());
let w2_dht = DhtNodeId::from_bytes(*w2_id.as_bytes());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let mut addr_map = HashMap::new();
addr_map.insert(w_dht, w_eth.address());
addr_map.insert(w2_dht, w2_eth.address());
addr_map.insert(a_dht, a_eth.address());
let attempted: Arc<Mutex<Vec<Address>>> = Arc::new(Mutex::new(Vec::new()));
let buyer = Arc::new(WedgedOpener {
wedged: HashSet::from([w_eth.address(), w2_eth.address()]),
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
attempted: Arc::clone(&attempted),
}) as Arc<dyn PoolOpener>;
let stall_budget = Duration::from_secs(20);
let per_candidate = Duration::from_secs(2);
let (origin, engine, _engine_tmp) = build_origin_seeded_ranking(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
&[(w_dht, STALL_RATE), (w2_dht, STALL_RATE), (a_dht, RATE)],
addr_map,
per_candidate,
stall_budget,
)
.await;
let fetched = tokio::time::timeout(
outer_pull_deadline(per_candidate, stall_budget),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("fetch never returned: the wedged opens starved the loop"))?
.map_err(|e| anyhow::anyhow!("node-origin fetch failed: {e}"))?;
anyhow::ensure!(
matches!(fetched, OriginFetch::AlreadyAdmitted),
"expected the blob from the honest fallback candidate"
);
let bytes = engine.get(hash).await?;
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"pulled bytes mismatch"
);
let attempted = attempted
.lock()
.map_err(|_| anyhow::anyhow!("attempted lock poisoned"))?
.clone();
anyhow::ensure!(
attempted.len() == 3 && attempted.last() == Some(&a_eth.address()),
"the honest candidate must be reached in the THIRD and last allowed attempt, after both \
wedged opens are abandoned; got {attempted:?}"
);
anyhow::ensure!(
attempted.iter().take(2).copied().collect::<HashSet<_>>()
== HashSet::from([w_eth.address(), w2_eth.address()]),
"both wedged candidates must be tried BEFORE the honest one, got {attempted:?}"
);
for (wedged_id, label) in [(w_id, "W1"), (w2_id, "W2")] {
anyhow::ensure!(
(local_rep.score(wedged_id) - 0.5).abs() < f64::EPSILON,
"{label}: the wedged provider's local score must stay neutral, got {}",
local_rep.score(wedged_id)
);
}
assert_counter(&b_metrics, "node_pull_success_total", 1)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
assert_counter(&b_metrics, "node_pull_pool_open_pending_total", 2)?;
assert_counter(&b_metrics, "node_pull_pool_open_failures_total", 0)?;
shutdown([task_a, task_w, task_w2], [&ep_b, &ep_a, &ep_w, &ep_w2]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_wedged_pool_open_does_not_starve_the_candidate_loop() -> Result<()> {
wedged_open_does_not_starve_the_candidate_loop().await
}
#[derive(Debug)]
struct ReportedFailureOpener {
local: bool,
}
#[async_trait]
impl PoolOpener for ReportedFailureOpener {
async fn open_or_reuse_pool(
&self,
_provider_addr: Address,
_budget: Duration,
) -> Result<PoolContext> {
let err = anyhow::anyhow!("open failed and was reported at its raising site")
.context(OpenReported);
Err(if self.local {
err.context(LocalPullFault)
} else {
err
})
}
fn record_progress(
&self,
_provider_addr: Address,
_pool_id: B256,
_write: decdn_client::buyer_pool::ProgressWrite,
) -> Result<()> {
Ok(())
}
}
async fn pull_through_a_reported_open_failure(local: bool) -> Result<Arc<Metrics>> {
let hash = Hash::new(b"reported open failure");
let provider_id = fresh_key().public();
let provider_dht = DhtNodeId::from_bytes(*provider_id.as_bytes());
let b_sk = fresh_key();
let b_dht = DhtNodeId::from_bytes(*b_sk.public().as_bytes());
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let addr_map = HashMap::from([(provider_dht, PrivateKeySigner::random().address())]);
let (origin, _engine, _engine_tmp) = build_origin_seeded_ranking(
&ep_b,
b_dht,
hash,
Arc::new(ReportedFailureOpener { local }) as Arc<dyn PoolOpener>,
&local_rep,
&b_metrics,
&[(provider_dht, RATE)],
addr_map,
Duration::from_secs(2),
Duration::from_secs(2),
)
.await;
let fetched = Origin::fetch(&origin, hash, u64::MAX).await;
anyhow::ensure!(
!matches!(fetched, Ok(OriginFetch::AlreadyAdmitted)),
"a pull whose only pool open failed cannot deliver"
);
ep_b.close().await;
Ok(b_metrics)
}
#[tokio::test(flavor = "multi_thread")]
async fn a_reported_local_open_fault_is_not_metered_again() -> Result<()> {
let metrics = pull_through_a_reported_open_failure(true).await?;
assert_counter(&metrics, "node_pull_pool_open_failures_total", 0)?;
assert_counter(&metrics, "node_pull_local_fault_total", 1)?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_reported_open_failure_is_not_metered_again() -> Result<()> {
let metrics = pull_through_a_reported_open_failure(false).await?;
assert_counter(&metrics, "node_pull_pool_open_failures_total", 0)?;
assert_counter(&metrics, "node_pull_local_fault_total", 0)?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn node_origin_no_providers_is_clean_miss() -> Result<()> {
let hash = Hash::new(b"nobody-has-this");
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (origin, _engine, recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
B256::repeat_byte(0xA1),
&Arc::new(PrivateKeySigner::random()),
&local_rep,
&b_metrics,
Vec::new(),
HashMap::new(),
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"no providers must yield NotFound"
);
anyhow::ensure!(
progress_log(&recorded)?.is_empty(),
"no pull ⇒ no voucher acked ⇒ nothing to persist (#852 guard)"
);
assert_counter(&b_metrics, "node_pull_no_providers_total", 1)?;
shutdown([], [&ep_b]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn an_origin_directory_entry_for_this_node_is_not_a_candidate() -> Result<()> {
let hash = Hash::new(b"only-this-node-is-listed");
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_dht = DhtNodeId::from_bytes(*fresh_key().public().as_bytes());
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (origin, _engine, _recorded, _engine_tmp) = provisioned_origin(
&ep_b,
b_dht,
hash,
B256::repeat_byte(0xA1),
&Arc::new(PrivateKeySigner::random()),
&local_rep,
&b_metrics,
vec![b_dht],
HashMap::new(),
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a directory listing only this node must yield NotFound"
);
assert_counter(&b_metrics, "node_pull_no_providers_total", 1)?;
shutdown([], [&ep_b]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn node_origin_unresolvable_address_skips_without_scoring() -> Result<()> {
let payload = vec![0x5Au8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) = local_endpoint(a_sk, vec![ALPN_PROBE.to_vec()]).await?;
let task_a = {
let ep_a = ep_a.clone();
let eth = Arc::clone(&a_eth);
tokio::spawn(async move {
while let Some(incoming) = ep_a.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(ð);
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &slash_domain(), RATE, total_bytes).await;
});
}
})
};
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (origin, _engine, recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
B256::repeat_byte(0xA1),
&Arc::new(PrivateKeySigner::random()),
&local_rep,
&b_metrics,
vec![DhtNodeId::from_bytes(*a_id.as_bytes())],
HashMap::new(),
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"unresolvable provider must yield NotFound"
);
anyhow::ensure!(
progress_log(&recorded)?.is_empty(),
"skipped-before-pull ⇒ nothing persisted (#852 guard)"
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn node_origin_probe_unreachable_is_scored() -> Result<()> {
let hash = Hash::new(b"present-at-a-dead-peer");
let a_id = fresh_key().public();
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let mut addr_map = HashMap::new();
addr_map.insert(
DhtNodeId::from_bytes(*a_id.as_bytes()),
Address::repeat_byte(0x44),
);
let (origin, _engine, recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
B256::repeat_byte(0xA1),
&Arc::new(PrivateKeySigner::random()),
&local_rep,
&b_metrics,
vec![DhtNodeId::from_bytes(*a_id.as_bytes())],
addr_map,
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"an unreachable provider must yield NotFound"
);
anyhow::ensure!(
progress_log(&recorded)?.is_empty(),
"an unprobeable provider is never paid ⇒ nothing persisted (#852 guard)"
);
assert_counter(&b_metrics, "node_pull_unreachable_total", 1)?;
shutdown([], [&ep_b]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn node_origin_corruption_is_classified_and_scored() -> Result<()> {
let honest = vec![0x11u8; 4096];
let hash = Hash::new(&honest);
let served = vec![0x22u8; 4096]; anyhow::ensure!(Hash::new(&served) != hash, "fixtures must differ");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let total_bytes = u64::try_from(served.len()).unwrap_or(u64::MAX);
let task_a = spawn_a_lying_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
served,
total_bytes,
RATE,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, _engine, recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
B256::repeat_byte(0xA1),
&b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"corrupt delivery must not surface bytes (NotFound)"
);
anyhow::ensure!(
local_rep.score(a_id) < 0.5,
"corruption must drop the local score below neutral, got {}",
local_rep.score(a_id)
);
assert_counter(&b_metrics, "node_pull_corruption_total", 1)?;
anyhow::ensure!(
progress_log(&recorded)?.is_empty(),
"a delivery with no verified byte must pay nothing, got {:?}",
progress_log(&recorded)?
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn node_origin_voucher_rejection_does_not_tar_upstream() -> Result<()> {
let payload = vec![0x33u8; 4096];
let hash = Hash::new(&payload);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let task_a = spawn_a_voucher_rejecting_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
payload.clone(),
total_bytes,
RATE,
VoucherRejectReason::AmountRegression,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, _engine, _recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
B256::repeat_byte(0xA1),
&b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a rejected voucher must not surface bytes (NotFound)"
);
anyhow::ensure!(
(local_rep.score(a_id) - 0.5).abs() < f64::EPSILON,
"provider score must stay neutral after a voucher rejection, got {}",
local_rep.score(a_id)
);
assert_counter(&b_metrics, "node_pull_attempts_total", 1)?;
assert_counter(&b_metrics, "node_pull_voucher_rejected_total", 1)?;
assert_counter(&b_metrics, "node_pull_success_total", 0)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
assert_counter(&b_metrics, "node_pull_corruption_total", 0)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
async fn pull_against_a_voucher_rejecting_upstream_n(
reason: VoucherRejectReason,
fetches: usize,
) -> Result<(Arc<Metrics>, Arc<LocalReputation>, iroh::PublicKey, B256)> {
let payload = vec![0x33u8; 4096];
let hash = Hash::new(&payload);
let pool_id = B256::repeat_byte(0xC7);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let task_a = spawn_a_voucher_rejecting_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
payload.clone(),
total_bytes,
RATE,
reason,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
)
.await;
let is_our_signer = matches!(
reason,
VoucherRejectReason::BadSignature | VoucherRejectReason::WrongSigner
);
for _ in 0..fetches {
let got = Origin::fetch(&origin, hash, u64::MAX).await;
if is_our_signer {
let err = got
.err()
.ok_or_else(|| anyhow::anyhow!("a signature the upstream cannot verify is OUR fault; the pull must not answer a clean miss"))?;
anyhow::ensure!(
!err.is_transient(),
"a broken buyer key is not cured by retrying — the engine must see a \
Permanent error, or it re-runs the pull and trips the peer origin's \
circuit breaker on our own defect, got {err:?}"
);
} else {
let got = got.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a rejected voucher must not surface bytes (NotFound)"
);
}
}
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok((b_metrics, local_rep, a_id, pool_id))
}
async fn pull_against_a_voucher_rejecting_upstream(
reason: VoucherRejectReason,
) -> Result<(Arc<Metrics>, Arc<LocalReputation>, iroh::PublicKey, B256)> {
pull_against_a_voucher_rejecting_upstream_n(reason, 1).await
}
#[tokio::test(flavor = "multi_thread")]
async fn node_origin_an_unverifiable_voucher_is_a_local_fault_not_a_payment_one() -> Result<()> {
let (metrics, local_rep, a_id, _) =
pull_against_a_voucher_rejecting_upstream(VoucherRejectReason::BadSignature).await?;
assert_counter(&metrics, "node_pull_local_fault_total", 1)?;
assert_counter(&metrics, "node_pull_voucher_rejected_total", 0)?;
assert_counter(&metrics, "node_pull_unreachable_total", 0)?;
anyhow::ensure!(
(local_rep.score(a_id) - 0.5).abs() < f64::EPSILON,
"provider score must stay neutral, got {}",
local_rep.score(a_id)
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_buffered_walk_that_exhausts_its_budget_on_local_faults_refuses_rather_than_missing()
-> Result<()> {
for candidates in [MAX_PROVIDER_ATTEMPTS, MAX_PROVIDER_ATTEMPTS - 1] {
buffered_local_fault_walk(candidates).await?;
}
Ok(())
}
async fn buffered_local_fault_walk(candidates: usize) -> Result<()> {
let payload = vec![0x5Du8; 4096];
let hash = Hash::new(&payload);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let mut addr_map = HashMap::new();
let ranked: Vec<(DhtNodeId, u64)> = (0..candidates)
.map(|_| {
let dht = DhtNodeId::from_bytes(*fresh_key().public().as_bytes());
addr_map.insert(dht, PrivateKeySigner::random().address());
(dht, RATE)
})
.collect();
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0x5D),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_seeded_ranking(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
&ranked,
addr_map,
Duration::from_secs(20),
Duration::ZERO,
)
.await;
let err = tokio::time::timeout(
Duration::from_secs(10),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("fetch never returned"))?
.err()
.ok_or_else(|| {
anyhow::anyhow!(
"a fetch whose whole budget went to OUR faults must not answer a clean miss"
)
})?;
anyhow::ensure!(
!err.is_transient(),
"a broken deadline config is not cured by retrying, got {err:?}"
);
assert_counter(
&b_metrics,
"node_pull_local_fault_total",
u64::try_from(candidates).unwrap_or(u64::MAX),
)?;
shutdown([], [&ep_b]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn a_local_fault_on_one_candidate_does_not_sink_a_walk_that_still_delivers() -> Result<()> {
const CHEAP_RATE: u64 = RATE / 2;
let payload = vec![0x5Cu8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let b_buyer = Arc::new(PrivateKeySigner::random());
let f_sk = fresh_key();
let f_id = f_sk.public();
let f_eth = Arc::new(PrivateKeySigner::random());
let (ep_f, addr_f) =
local_endpoint(f_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_f = spawn_a_voucher_rejecting_server(
ep_f.clone(),
Arc::clone(&f_eth),
slash_domain(),
payload.clone(),
total_bytes,
CHEAP_RATE,
VoucherRejectReason::BadSignature,
);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0x5C);
let (cache_a, hash_a, tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
std::mem::forget(tmp_a);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let a_metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&a_metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&a_metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for (id, addr) in [(f_id, addr_f), (a_id, addr_a)] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(id).with_ip_addr(addr),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
let f_dht = DhtNodeId::from_bytes(*f_id.as_bytes());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let mut addr_map = HashMap::new();
addr_map.insert(f_dht, f_eth.address());
addr_map.insert(a_dht, a_eth.address());
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_seeded_ranking(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
&[(f_dht, CHEAP_RATE), (a_dht, RATE)],
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
)
.await;
let got = tokio::time::timeout(
Duration::from_secs(20),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("fetch never returned"))?
.map_err(|e| anyhow::anyhow!("a walk that reached an honest candidate must deliver: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::AlreadyAdmitted),
"the honest fallback held the blob; the walk must not miss"
);
let delivered = engine.get(hash).await?;
anyhow::ensure!(
delivered.as_ref() == payload.as_slice(),
"the fallback candidate's bytes must be the ones returned"
);
assert_counter(&b_metrics, "node_pull_local_fault_total", 1)?;
assert_counter(&b_metrics, "node_pull_success_total", 1)?;
shutdown([task_a, task_f], [&ep_b, &ep_a, &ep_f]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn node_origin_transport_failure_still_scores_unreachable() -> Result<()> {
let payload = vec![0x44u8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_probe_ok_client_dead_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, _engine, _recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
B256::repeat_byte(0xA1),
&b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a dead client stream must not surface bytes (NotFound)"
);
anyhow::ensure!(
local_rep.score(a_id) < 0.5,
"a transport failure must drop the local score below neutral, got {}",
local_rep.score(a_id)
);
assert_counter(&b_metrics, "node_pull_unreachable_total", 1)?;
assert_counter(&b_metrics, "node_pull_timeout_total", 0)?;
assert_counter(&b_metrics, "node_pull_voucher_rejected_total", 0)?;
assert_counter(&b_metrics, "node_pull_corruption_total", 0)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
async fn read_stream_request(recv: &mut iroh::endpoint::RecvStream) -> Result<StreamRequest> {
let frame = read_frame(recv)
.await
.map_err(|e| anyhow::anyhow!("read request: {e}"))?;
let (msg, _) = decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode request: {e}"))?;
let ClientMessage::StreamRequest(req) = msg else {
anyhow::bail!("expected a StreamRequest");
};
Ok(req)
}
fn signed_response(
req: &StreamRequest,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
rate: u64,
total_bytes: u64,
error: Option<StreamError>,
) -> Result<(StreamResponse, StreamResponseExt)> {
let body = StreamResponseBody {
hash: req.hash,
ok: error.is_none(),
rate_per_mb: rate,
total_bytes,
pool_id: req.pool_id,
timestamp_us: req.timestamp_us,
};
let slash_sig = StreamSlashData::from_response_body(&body)
.sign(eth.as_ref(), slash)
.map_err(|e| anyhow::anyhow!("slash sign: {e}"))?
.as_bytes()
.to_vec();
Ok((
StreamResponse { body, slash_sig },
StreamResponseExt { error },
))
}
fn honest_bao_wire(payload: &[u8]) -> Result<Vec<u8>> {
support::honest_bao_wire_range(payload, 0, 0)
}
fn encode_empty_chunk_frame() -> Result<Vec<u8>> {
let mut frame = encode_message(&ClientMessage::ChunkData(ChunkData::new(vec![0x00])?))?;
anyhow::ensure!(
frame.len() >= 2 && frame.pop() == Some(0x00) && frame.pop() == Some(0x01),
"expected a one-byte ChunkData to encode as <variant><len=1><byte>; \
the framing changed and this forgery needs rewriting"
);
frame.push(0x00); Ok(frame)
}
async fn serve_empty_chunks(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
total_bytes: u64,
rate: u64,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req = read_stream_request(&mut recv).await?;
let (resp, resp_ext) = signed_response(&req, eth, slash, rate, total_bytes, None)?;
write_frame(&mut send, &encode_stream_response(&resp, Some(&resp_ext))?)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
let empty = encode_empty_chunk_frame()?;
loop {
tokio::time::sleep(EMPTY_CHUNK_GAP).await;
if write_frame(&mut send, &empty).await.is_err() {
break;
}
}
Ok(())
}
fn spawn_an_empty_chunk_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
total_bytes: u64,
rate: u64,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
tokio::spawn(async move {
let _ = serve_empty_chunks(conn, ð, &dom, total_bytes, rate).await;
});
}
}
})
}
async fn serve_then_go_silent(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
wire: Arc<Vec<u8>>,
total_bytes: u64,
rate: u64,
prefix_chunks: usize,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req = read_stream_request(&mut recv).await?;
let (resp, resp_ext) = signed_response(&req, eth, slash, rate, total_bytes, None)?;
write_frame(&mut send, &encode_stream_response(&resp, Some(&resp_ext))?)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
for frame in wire_frames(&wire, prefix_chunks)? {
write_frame(&mut send, &frame)
.await
.map_err(|e| anyhow::anyhow!("write chunk: {e}"))?;
}
conn.closed().await;
Ok(())
}
fn wire_frames(wire: &[u8], chunks: usize) -> Result<Vec<Vec<u8>>> {
wire.chunks(WIRE_FRAME)
.take(chunks)
.map(|c| {
let data = ChunkData::new(c.to_vec())?;
Ok(encode_message(&ClientMessage::ChunkData(data))?)
})
.collect()
}
fn spawn_a_mid_stream_silent_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
wire: Arc<Vec<u8>>,
total_bytes: u64,
rate: u64,
prefix_chunks: usize,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let wire = Arc::clone(&wire);
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
tokio::spawn(async move {
let _ = serve_then_go_silent(
conn,
ð,
&dom,
wire,
total_bytes,
rate,
prefix_chunks,
)
.await;
});
}
}
})
}
async fn serve_a_paid_interval_then_go_silent(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
wire: Arc<Vec<u8>>,
total_bytes: u64,
rate: u64,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req = read_stream_request(&mut recv).await?;
let (resp, resp_ext) = signed_response(&req, eth, slash, rate, total_bytes, None)?;
write_frame(&mut send, &encode_stream_response(&resp, Some(&resp_ext))?)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
let interval_bytes =
usize::try_from(CHUNK_BYTES + decdn_client::VERIFY_LAG_BYTES).unwrap_or(usize::MAX);
let interval_chunks = interval_bytes.div_ceil(WIRE_FRAME);
for frame in wire_frames(&wire, interval_chunks)? {
write_frame(&mut send, &frame)
.await
.map_err(|e| anyhow::anyhow!("write chunk: {e}"))?;
}
let voucher_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read voucher: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode voucher: {e}"))?
.0
};
let ClientMessage::Voucher(_) = voucher_msg else {
anyhow::bail!("expected a Voucher at the interval boundary, got {voucher_msg:?}");
};
conn.closed().await;
Ok(())
}
fn spawn_a_paid_then_silent_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
wire: Arc<Vec<u8>>,
total_bytes: u64,
rate: u64,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let wire = Arc::clone(&wire);
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
tokio::spawn(async move {
let _ = serve_a_paid_interval_then_go_silent(
conn,
ð,
&dom,
wire,
total_bytes,
rate,
)
.await;
});
}
}
})
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn node_origin_cancelled_pull_still_persists_the_acked_watermark() -> Result<()> {
let payload_len = usize::try_from(CHUNK_BYTES.saturating_mul(2)).unwrap_or(usize::MAX);
let payload = vec![0x7Du8; payload_len];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload_len).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_paid_then_silent_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
Arc::new(honest_bao_wire(&payload)?),
total_bytes,
RATE,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let recorded = Arc::new(Mutex::new(Vec::new()));
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0xD2),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::clone(&recorded),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_mins(2),
0,
)
.await;
let cancelled = tokio::time::timeout(
Duration::from_secs(5),
Origin::fetch(&origin, hash, u64::MAX),
)
.await;
anyhow::ensure!(
cancelled.is_err(),
"fixture is wrong: the pull must still have been in flight (waiting on a \
silent-but-paid upstream) when the cancellation dropped it"
);
let interval_amount = min_payment(CHUNK_BYTES, RATE);
let want = vec![(a_eth.address(), U256::from(CHUNK_BYTES), interval_amount)];
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
while progress_log(&recorded)? != want {
anyhow::ensure!(
tokio::time::Instant::now() < deadline,
"a cancelled pull must persist the watermark the upstream already acked, \
from the pull thread — otherwise the next reuse re-signs from a stale \
cumulative amount and the lane wedges, suppressing the provider. Got {:?}",
progress_log(&recorded)?
);
tokio::time::sleep(Duration::from_millis(50)).await;
}
tokio::time::sleep(Duration::from_secs(1)).await;
anyhow::ensure!(
progress_log(&recorded)? == want,
"a cancelled pull must settle once at the acked watermark and not keep \
advancing after cancel; got {:?}",
progress_log(&recorded)?
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
async fn serve_then_error_mid_stream(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
wire: Arc<Vec<u8>>,
total_bytes: u64,
rate: u64,
error: StreamError,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req = read_stream_request(&mut recv).await?;
let (resp, resp_ext) = signed_response(&req, eth, slash, rate, total_bytes, None)?;
write_frame(&mut send, &encode_stream_response(&resp, Some(&resp_ext))?)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
for frame in wire_frames(&wire, 2)? {
write_frame(&mut send, &frame)
.await
.map_err(|e| anyhow::anyhow!("write chunk: {e}"))?;
}
write_frame(
&mut send,
&encode_message(&ClientMessage::StreamError(error))?,
)
.await
.map_err(|e| anyhow::anyhow!("write stream error: {e}"))?;
let _ = send.finish();
conn.closed().await;
Ok(())
}
fn spawn_a_mid_stream_error_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
wire: Arc<Vec<u8>>,
total_bytes: u64,
rate: u64,
error: StreamError,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let err = error.clone();
let wire = Arc::clone(&wire);
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
tokio::spawn(async move {
let _ =
serve_then_error_mid_stream(conn, ð, &dom, wire, total_bytes, rate, err)
.await;
});
}
}
})
}
fn spawn_a_refusing_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
total_bytes: u64,
rate: u64,
error: StreamError,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let error = error.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
tokio::spawn(async move {
let _ = serve_refusal(conn, ð, &dom, total_bytes, rate, error).await;
});
}
}
})
}
#[allow(clippy::too_many_arguments)]
fn spawn_a_refusing_server_with_counters(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
total_bytes: u64,
rate: u64,
error: StreamError,
probes: Arc<AtomicUsize>,
streams: Arc<AtomicUsize>,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let error = error.clone();
if conn.alpn() == ALPN_PROBE {
let probes = Arc::clone(&probes);
tokio::spawn(async move {
probes.fetch_add(1, Ordering::SeqCst);
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
let streams = Arc::clone(&streams);
tokio::spawn(async move {
streams.fetch_add(1, Ordering::SeqCst);
let _ = serve_refusal(conn, ð, &dom, total_bytes, rate, error).await;
});
}
}
})
}
async fn serve_refusal(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
total_bytes: u64,
rate: u64,
error: StreamError,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req = read_stream_request(&mut recv).await?;
let (resp, resp_ext) = signed_response(&req, eth, slash, rate, total_bytes, Some(error))?;
write_frame(&mut send, &encode_stream_response(&resp, Some(&resp_ext))?)
.await
.map_err(|e| anyhow::anyhow!("write refusal: {e}"))?;
let _ = send.finish();
conn.closed().await;
Ok(())
}
fn spawn_a_slow_but_healthy_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
wire: Vec<u8>,
total_bytes: u64,
rate: u64,
gap: Duration,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let wire = wire.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
tokio::spawn(async move {
let _ = serve_wire_paced(conn, ð, &dom, &wire, total_bytes, rate, gap).await;
});
}
}
})
}
const EMPTY_CHUNK_GAP: Duration = Duration::from_millis(1);
const EMPTY_CHUNK_STALL_BUDGET: Duration = Duration::from_secs(30);
const EMPTY_CHUNK_ASSERT_WINDOW: Duration = Duration::from_secs(10);
#[tokio::test(flavor = "multi_thread")]
async fn node_origin_empty_chunk_stream_is_rejected_not_spun_on() -> Result<()> {
let payload = vec![0x5Eu8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_an_empty_chunk_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0xE8),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(10),
EMPTY_CHUNK_STALL_BUDGET,
0,
)
.await;
let got = tokio::time::timeout(
EMPTY_CHUNK_ASSERT_WINDOW,
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| {
anyhow::anyhow!(
"the receive loop never returned: it spun on empty ChunkData frames, \
which advance neither the byte total nor the voucher cadence (#1088)"
)
})?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"an empty-chunk stream must not surface bytes (NotFound)"
);
assert_counter(&b_metrics, "node_pull_attempts_total", 1)?;
assert_counter(&b_metrics, "node_pull_stalled_total", 0)?;
assert_counter(&b_metrics, "node_pull_success_total", 0)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn node_origin_mid_stream_silence_does_not_score_stalled_upstream() -> Result<()> {
let payload = vec![0x7Du8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_mid_stream_silent_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
Arc::new(honest_bao_wire(&payload)?),
total_bytes,
RATE,
3,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0xD1),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(2),
0,
)
.await;
let got = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("a mid-stream-silent upstream was never abandoned (#1134)"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a stalled delivery must not surface bytes (NotFound)"
);
assert_counter(&b_metrics, "node_pull_stalled_total", 1)?;
assert_counter(&b_metrics, "node_pull_timeout_total", 0)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
assert_counter(&b_metrics, "node_pull_voucher_rejected_total", 0)?;
assert_counter(&b_metrics, "node_pull_corruption_total", 0)?;
anyhow::ensure!(
(local_rep.score(a_id) - 0.5).abs() < 1e-9,
"a mid-stream stall must NOT move the local score off neutral (#1797), got {}",
local_rep.score(a_id)
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn node_origin_a_silent_first_byte_is_our_deadline_not_the_peers_fault() -> Result<()> {
let payload = vec![0x7Eu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_mid_stream_silent_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
Arc::new(honest_bao_wire(&payload)?),
total_bytes,
RATE,
0,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0x7E),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(2),
0,
)
.await;
let got = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("a silent upstream was never abandoned"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a first-byte timeout must not surface bytes (NotFound)"
);
assert_counter(&b_metrics, "node_pull_timeout_total", 1)?;
assert_counter(&b_metrics, "node_pull_stalled_total", 0)?;
let again = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("the second fetch never returned"))?
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
anyhow::ensure!(matches!(again, OriginFetch::NotFound), "still no bytes");
assert_counter(&b_metrics, "node_pull_timeout_total", 1)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
anyhow::ensure!(
(local_rep.score(a_id) - 0.5).abs() < f64::EPSILON,
"the local score must stay neutral, got {}",
local_rep.score(a_id)
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn node_origin_mid_stream_refusal_is_metered_not_scored() -> Result<()> {
let payload = vec![0x8Bu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_mid_stream_error_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
Arc::new(honest_bao_wire(&payload)?),
total_bytes,
RATE,
StreamError::NotFound,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0x8B),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
)
.await;
let got = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("a mid-stream refusal never ended the pull"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a refused delivery must not surface bytes (NotFound)"
);
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
assert_counter(&b_metrics, "node_pull_stalled_total", 0)?;
anyhow::ensure!(
local_rep.score(a_id) >= 0.5,
"an honest mid-stream refusal must leave the local score neutral, got {}",
local_rep.score(a_id)
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn node_origin_an_ack_wait_refusal_is_metered_not_scored() -> Result<()> {
let payload = vec![0x4Du8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_voucher_erroring_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
payload.clone(),
total_bytes,
RATE,
StreamError::NotFound,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0x4D),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
)
.await;
let got = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("an ack-wait refusal never ended the pull"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a refused delivery must not surface bytes (NotFound)"
);
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
assert_counter(&b_metrics, "node_pull_stalled_total", 0)?;
anyhow::ensure!(
local_rep.score(a_id) >= 0.5,
"an honest ack-wait refusal must leave the local score neutral, got {}",
local_rep.score(a_id)
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn node_origin_a_wedged_provider_is_skipped_for_other_hashes() -> Result<()> {
let payload1 = vec![0x11u8; 4096];
let payload2 = vec![0x22u8; 4096];
let hash1 = Hash::new(&payload1);
let hash2 = Hash::new(&payload2);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_voucher_rejecting_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
payload1.clone(),
u64::try_from(payload1.len()).unwrap_or(u64::MAX),
RATE,
VoucherRejectReason::AmountRegression,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let b_dht = DhtNodeId::from_bytes(*b_id.as_bytes());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
for h in [&hash1, &hash2] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*h.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0xA1),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let mut addr_map = HashMap::new();
addr_map.insert(a_dht, a_eth.address());
let (origin, _engine, _engine_tmp) = build_origin_multi_hash(
&ep_b,
b_dht,
&[hash1, hash2],
buyer,
&local_rep,
&b_metrics,
&[a_dht],
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
)
.await;
let got1 = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash1, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("pull 1 never ended"))?
.map_err(|e| anyhow::anyhow!("pull 1: {e}"))?;
anyhow::ensure!(matches!(got1, OriginFetch::NotFound), "pull 1 must refuse");
assert_counter(&b_metrics, "node_pull_pool_wedged_total", 1)?;
let got2 = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash2, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("pull 2 never ended"))?
.map_err(|e| anyhow::anyhow!("pull 2: {e}"))?;
anyhow::ensure!(
matches!(got2, OriginFetch::NotFound),
"pull 2 must find no candidate"
);
assert_counter(&b_metrics, "node_pull_pool_wedged_total", 1)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn node_origin_a_mid_stream_voucher_rejection_still_reaches_the_lane_remedy() -> Result<()> {
let payload = vec![0x9Cu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_mid_stream_error_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
Arc::new(honest_bao_wire(&payload)?),
total_bytes,
RATE,
StreamError::VoucherRejected {
reason: VoucherRejectReason::SpendingCapExhausted,
bundle: None,
},
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0x9C),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
)
.await;
let got = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("a mid-stream voucher rejection never ended the pull"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a rejected voucher must not surface bytes (NotFound)"
);
assert_counter(&b_metrics, "node_pull_pool_wedged_total", 1)?;
assert_counter(&b_metrics, "node_pull_voucher_rejected_total", 1)?;
assert_counter(&b_metrics, "node_pull_refused_total", 0)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
anyhow::ensure!(
(local_rep.score(a_id) - 0.5).abs() < f64::EPSILON,
"the provider's score must stay neutral, got {}",
local_rep.score(a_id)
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn silent_upstreams_do_not_starve_the_candidate_loop() -> Result<()> {
let per_candidate = Duration::from_secs(5);
let stall_budget = Duration::from_secs(5);
let payload = vec![0x5Eu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let s1_sk = fresh_key();
let s1_id = s1_sk.public();
let s1_eth = Arc::new(PrivateKeySigner::random());
let (ep_s1, addr_s1) =
local_endpoint(s1_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let wire = Arc::new(honest_bao_wire(&payload)?);
let task_s1 = spawn_a_mid_stream_silent_server(
ep_s1.clone(),
Arc::clone(&s1_eth),
slash_domain(),
Arc::clone(&wire),
total_bytes,
STALL_RATE,
3,
);
let s2_sk = fresh_key();
let s2_id = s2_sk.public();
let s2_eth = Arc::new(PrivateKeySigner::random());
let (ep_s2, addr_s2) =
local_endpoint(s2_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_s2 = spawn_a_mid_stream_silent_server(
ep_s2.clone(),
Arc::clone(&s2_eth),
slash_domain(),
Arc::clone(&wire),
total_bytes,
STALL_RATE,
3,
);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xC7);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let a_metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&a_metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&a_metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for (id, addr) in [(s1_id, addr_s1), (s2_id, addr_s2), (a_id, addr_a)] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(id).with_ip_addr(addr),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let s1_dht = DhtNodeId::from_bytes(*s1_id.as_bytes());
let s2_dht = DhtNodeId::from_bytes(*s2_id.as_bytes());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let mut addr_map = HashMap::new();
addr_map.insert(s1_dht, s1_eth.address());
addr_map.insert(s2_dht, s2_eth.address());
addr_map.insert(a_dht, a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_seeded_ranking(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
&[(s1_dht, STALL_RATE), (s2_dht, STALL_RATE), (a_dht, RATE)],
addr_map,
per_candidate,
stall_budget,
)
.await;
let fetched = tokio::time::timeout(
outer_pull_deadline(per_candidate, stall_budget),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| {
anyhow::anyhow!(
"the outer deadline expired before the loop could try every candidate — the \
stall window is a per-candidate cost and `outer_pull_deadline` must budget it"
)
})?
.map_err(|e| anyhow::anyhow!("node-origin fetch failed: {e}"))?;
anyhow::ensure!(
matches!(fetched, OriginFetch::AlreadyAdmitted),
"the loop gave up before reaching the healthy candidate #3 — two silent \
candidates consumed a budget the outer deadline had not allowed for"
);
let bytes = engine.get(hash).await?;
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"wrong bytes from candidate #3"
);
{
let deadline = tokio::time::Instant::now() + Duration::from_millis(800);
let mut v = counter_value(&b_metrics, "node_pull_stalled_total")?;
while v != 2 && tokio::time::Instant::now() < deadline {
tokio::time::sleep(Duration::from_millis(50)).await;
v = counter_value(&b_metrics, "node_pull_stalled_total")?;
}
}
assert_counter(&b_metrics, "node_pull_stalled_total", 2)?;
assert_counter(&b_metrics, "node_pull_timeout_total", 0)?;
shutdown([task_a, task_s1, task_s2], [&ep_b, &ep_a, &ep_s1, &ep_s2]).await?;
Ok(())
}
const SLOW_PULL_OPEN_BUDGET: Duration = Duration::from_secs(2);
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn node_origin_slow_but_healthy_transfer_completes_past_pull_timeout() -> Result<()> {
let payload = vec![0x51u8; 6 * WIRE_FRAME];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let wire = honest_bao_wire(&payload)?;
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let gap = Duration::from_millis(600);
let task_a = spawn_a_slow_but_healthy_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
wire,
total_bytes,
RATE,
gap,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0x51),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
SLOW_PULL_OPEN_BUDGET,
Duration::from_secs(5),
0,
)
.await;
let started = std::time::Instant::now();
let fetched = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("a slow-but-healthy pull must complete, got: {e}"))?;
anyhow::ensure!(
matches!(fetched, OriginFetch::AlreadyAdmitted),
"a slow-but-healthy pull returned NotFound: the transfer was killed by a \
whole-blob deadline it must not have (#1134)"
);
let bytes = engine.get(hash).await?;
let elapsed = started.elapsed();
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"slow-pull bytes mismatch"
);
anyhow::ensure!(
elapsed > SLOW_PULL_OPEN_BUDGET,
"the paced transfer finished in {elapsed:?}, inside the {SLOW_PULL_OPEN_BUDGET:?} \
budget, so a whole-blob deadline at that budget would not trip and this test \
cannot detect the regression"
);
assert_counter(&b_metrics, "node_pull_timeout_total", 0)?;
assert_counter(&b_metrics, "node_pull_stalled_total", 0)?;
assert_counter(&b_metrics, "node_pull_success_total", 1)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn node_origin_not_found_refusal_does_not_tar_upstream() -> Result<()> {
let payload = vec![0x4Eu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0x4E);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let (cache_n, _tmp_n) = empty_cache().await?;
let n_sk = fresh_key();
let n_id = n_sk.public();
let n_eth = Arc::new(PrivateKeySigner::random());
let store_n = Arc::new(MemoryPoolStateStore::new());
store_n.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
n_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_n = Arc::new(Metrics::new());
let handler_n = build_handler_full(
n_id,
&n_eth,
&metrics_n,
permissive_limiter(&metrics_n),
cache_n,
store_n as Arc<dyn PoolStateStore>,
STALL_RATE,
&domains,
16,
)?;
let (ep_n, addr_n) =
local_endpoint(n_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_n = spawn_a_server(
ep_n.clone(),
handler_n,
Arc::clone(&n_eth),
slash_domain(),
total_bytes,
STALL_RATE,
);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_a = Arc::new(Metrics::new());
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics_a,
permissive_limiter(&metrics_a),
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for (id, addr) in [(n_id, addr_n), (a_id, addr_a)] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(id).with_ip_addr(addr),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let n_dht = DhtNodeId::from_bytes(*n_id.as_bytes());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let mut addr_map = HashMap::new();
addr_map.insert(n_dht, n_eth.address());
addr_map.insert(a_dht, a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_seeded_ranking(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
&[(n_dht, STALL_RATE), (a_dht, RATE)],
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
)
.await;
let fetched = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("node-origin fetch failed: {e}"))?;
anyhow::ensure!(
matches!(fetched, OriginFetch::AlreadyAdmitted),
"expected the blob from the candidate that holds it"
);
let bytes = engine.get(hash).await?;
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"pulled bytes mismatch"
);
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
assert_counter(&b_metrics, "node_pull_success_total", 1)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
anyhow::ensure!(
(local_rep.score(n_id) - 0.5).abs() < f64::EPSILON,
"an honest NotFound must leave the refusing node's local score neutral, got {}",
local_rep.score(n_id)
);
anyhow::ensure!(
local_rep.score(a_id) > 0.5,
"A's clean delivery must raise its local score, got {}",
local_rep.score(a_id)
);
let refetched = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second node-origin fetch failed: {e}"))?;
anyhow::ensure!(
matches!(refetched, OriginFetch::AlreadyAdmitted)
&& engine.get(hash).await?.as_ref() == payload.as_slice(),
"the second fetch must still deliver the blob from A"
);
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
assert_counter(&b_metrics, "node_pull_success_total", 2)?;
shutdown([task_n, task_a], [&ep_b, &ep_n, &ep_a]).await?;
Ok(())
}
const TINY_CACHE_TTL: Duration = Duration::from_millis(100);
const PAST_THE_TINY_TTL: Duration = Duration::from_millis(600);
async fn refusal_suppression_after(error: StreamError, wait: Duration) -> Result<bool> {
let payload = vec![0x6Bu8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_refusing_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
error,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0x6B),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_negative_cache(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
NegativeProbeCache::with_capacity_and_ttl(16, TINY_CACHE_TTL),
)
.await;
let _ = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch: {e}"))?;
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
tokio::time::sleep(wait).await;
let _ = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
let text = b_metrics
.encode()
.map_err(|e| anyhow::anyhow!("encode metrics: {e}"))?;
let refused_twice = text.lines().any(|l| l == "decdn_node_pull_refused_total 2");
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(!refused_twice)
}
#[tokio::test(flavor = "multi_thread")]
async fn a_transient_refusal_is_suppressed_briefly_and_a_durable_one_for_the_full_ttl() -> Result<()>
{
anyhow::ensure!(
refusal_suppression_after(StreamError::NotFound, PAST_THE_TINY_TTL).await?,
"a transient refusal must be suppressed on REFUSAL_SUPPRESSION_TTL, not the cache TTL \
— routing it through the cache-wide TTL is exactly the bug that blackholed a healthy \
upstream for five minutes over one dry deposit"
);
anyhow::ensure!(
refusal_suppression_after(StreamError::Declined, Duration::ZERO).await?,
"a durable refusal must suppress the (peer, hash) pair — otherwise a peer that \
advertises everything and serves nothing keeps winning the ranker and burns a \
MAX_PROVIDER_ATTEMPTS slot on every miss, forever"
);
anyhow::ensure!(
!refusal_suppression_after(StreamError::Declined, PAST_THE_TINY_TTL).await?,
"a durable refusal must ride the cache-wide TTL; outliving one this long means it is \
being given the transient budget instead"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn node_origin_declined_refusal_is_not_scored() -> Result<()> {
let payload = vec![0x1Eu8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_refusing_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
StreamError::Declined,
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, _engine, _recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
B256::repeat_byte(0x1E),
&b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a refused delivery must not surface bytes (NotFound)"
);
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
anyhow::ensure!(
(local_rep.score(a_id) - 0.5).abs() < f64::EPSILON,
"a Declined refusal must leave the local score neutral, got {}",
local_rep.score(a_id)
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn node_origin_oversized_blob_aborts_on_received_bytes_without_scoring() -> Result<()> {
let payload = vec![0xABu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let ceiling: u64 = 1_048_576;
anyhow::ensure!(total_bytes > ceiling, "fixture must exceed the ceiling");
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xA1);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, _engine, _engine_tmp) = provisioned_origin_with_ceiling(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
pool_id,
&b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
ceiling,
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"an over-ceiling blob must abort the fill and surface no bytes (NotFound)"
);
anyhow::ensure!(
(local_rep.score(a_id) - 0.5).abs() < f64::EPSILON,
"provider score must stay neutral after a ceiling rejection, got {}",
local_rep.score(a_id)
);
assert_counter(&b_metrics, "node_pull_attempts_total", 1)?;
assert_counter(&b_metrics, "node_pull_too_large_total", 1)?;
assert_counter(&b_metrics, "node_pull_success_total", 0)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
assert_counter(&b_metrics, "node_pull_corruption_total", 0)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn node_origin_over_ceiling_rate_is_rejected_without_scoring() -> Result<()> {
let payload = vec![0xABu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let probe_rate = RATE / 2;
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xA2);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
probe_rate,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, _engine, _engine_tmp) = provisioned_origin_with_ceiling(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
pool_id,
&b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
0,
)
.await;
let got = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"an over-ceiling quote must not surface bytes (NotFound)"
);
anyhow::ensure!(
(local_rep.score(a_id) - 0.5).abs() < f64::EPSILON,
"provider score must stay neutral after a rate-ceiling rejection, got {}",
local_rep.score(a_id)
);
assert_counter(&b_metrics, "node_pull_attempts_total", 1)?;
assert_counter(&b_metrics, "node_pull_rate_above_ceiling_total", 1)?;
assert_counter(&b_metrics, "node_pull_success_total", 0)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
assert_counter(&b_metrics, "node_pull_corruption_total", 0)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn node_origin_reused_lane_resumes_voucher_progress() -> Result<()> {
let payload1 = vec![0xABu8; PAYLOAD_LEN];
let payload2 = vec![0xCDu8; PAYLOAD_LEN];
let hash1 = Hash::new(&payload1);
let hash2 = Hash::new(&payload2);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, _tmp_a) = cache_with_blobs(&[&payload1, &payload2]).await?;
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xA1);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for hash in [hash1, hash2] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let (_providers, addr_map) = one_provider(a_dht, a_eth.address());
let recorded: Arc<Mutex<Vec<ProgressEntry>>> = Arc::new(Mutex::new(Vec::new()));
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::clone(&recorded),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_multi_hash(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
&[hash1, hash2],
buyer,
&local_rep,
&b_metrics,
&[a_dht],
addr_map,
DEFAULT_TEST_PULL_DEADLINES.0,
DEFAULT_TEST_PULL_DEADLINES.1,
)
.await;
let first_fetch = Origin::fetch(&origin, hash1, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch failed: {e}"))?;
anyhow::ensure!(
matches!(first_fetch, OriginFetch::AlreadyAdmitted),
"first fetch returned NotFound"
);
let first = engine.get(hash1).await?;
anyhow::ensure!(
first.as_ref() == payload1.as_slice(),
"first pull bytes mismatch"
);
let second_fetch = Origin::fetch(&origin, hash2, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second fetch failed: {e}"))?;
anyhow::ensure!(
matches!(second_fetch, OriginFetch::AlreadyAdmitted),
"second fetch returned NotFound — stale voucher rejected (the #852 bug)"
);
let second = engine.get(hash2).await?;
anyhow::ensure!(
second.as_ref() == payload2.as_slice(),
"second pull bytes mismatch"
);
let expected_wire =
decdn_cache::range_pull::bao_encoded_size(total_bytes, &bao_tree::ChunkRanges::all());
let expected_amount = U256::from(expected_wire)
.saturating_mul(U256::from(RATE))
.div_ceil(U256::from(MB_BYTES));
let log = progress_log(&recorded)?;
anyhow::ensure!(
log == vec![
(a_eth.address(), U256::from(expected_wire), expected_amount),
(
a_eth.address(),
U256::from(expected_wire).saturating_mul(U256::from(2)),
expected_amount.saturating_mul(U256::from(2)),
),
],
"expected two monotonically-advancing progress entries, got {log:?}"
);
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn node_origin_persist_failure_still_delivers_and_is_counted() -> Result<()> {
let payload = vec![0xABu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xA1);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let buyer = Arc::new(FailingRecordOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
)
.await;
let fetched = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("fetch failed: {e}"))?;
anyhow::ensure!(
matches!(fetched, OriginFetch::AlreadyAdmitted),
"persist failure must not turn the pull into NotFound"
);
let bytes = engine.get(hash).await?;
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"delivered bytes mismatch"
);
assert_counter(&b_metrics, "node_pull_success_total", 1)?;
assert_counter(&b_metrics, "node_pull_progress_persist_failures_total", 1)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[derive(Debug)]
struct LeafOutcome {
received: u64,
acks: u64,
paid_wire: u64,
completed: bool,
hash_ok: bool,
}
#[derive(Debug, Clone, Copy)]
enum LeafMode {
PayAll,
DropAfter(u64),
StopPayingAfter { acks: u64, hold: Duration },
}
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
async fn leaf_ranged_paid_pull(
leaf_ep: &iroh::Endpoint,
target: EndpointAddr,
leaf_node_id: B256,
leaf_eth: &Arc<PrivateKeySigner>,
provider: Address,
pool_id: B256,
hash: Hash,
byte_offset: u64,
byte_len: u64,
rate: u64,
) -> Result<Vec<u8>> {
use alloy::signers::SignerSync;
let conn = leaf_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("leaf connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("leaf open_bi: {e}"))?;
let binding_hash = binding_signing_hash(leaf_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = leaf_eth.sign_hash_sync(&binding_hash)?.as_bytes().to_vec();
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: leaf_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset,
byte_len,
timestamp_us: 0x9008,
};
let payload =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame(&mut send, &payload)
.await
.map_err(|e| anyhow::anyhow!("write req: {e}"))?;
let (resp, resp_ext) = read_client_response(&mut recv).await?;
anyhow::ensure!(resp.body.ok, "delivery refused: {:?}", resp_ext.error);
let total = resp.body.total_bytes;
let aligned = decdn_cache::range_pull::align_range_clamped(byte_offset, byte_len, total)
.map_err(|e| anyhow::anyhow!("align range: {e}"))?;
let expected_wire = decdn_cache::range_pull::bao_encoded_size(total, aligned.chunk_ranges());
let interval_bytes = CHUNK_BYTES;
let mut buf = BytesMut::new();
let mut cumulative: u64 = 0;
let mut unvouchered: u64 = 0;
loop {
match read_client(&mut recv).await? {
ClientMessage::ChunkData(chunk) => {
buf.extend_from_slice(chunk.bytes());
let len = u64::try_from(chunk.bytes().len()).unwrap_or(u64::MAX);
cumulative = cumulative.saturating_add(len);
unvouchered = unvouchered.saturating_add(len);
let boundary = interval_bytes > 0 && unvouchered >= interval_bytes;
let closing = cumulative >= expected_wire && unvouchered > 0;
if boundary || closing {
let amount = U256::from(cumulative)
.saturating_mul(U256::from(rate))
.div_ceil(U256::from(MB_BYTES));
let signed = Voucher {
pool_id,
signer: leaf_eth.address(),
provider,
amount,
bytes_delivered: U256::from(cumulative),
chain_root: B256::ZERO,
chunk_price: U256::ZERO,
}
.sign(leaf_eth.as_ref(), &voucher_dom())
.map_err(|e| anyhow::anyhow!("sign voucher: {e}"))?;
write_client(
&mut send,
&ClientMessage::Voucher(signed_to_wire_voucher(&signed)?),
)
.await?;
unvouchered = 0;
}
}
ClientMessage::StreamEnd => break,
ClientMessage::StreamError(e) => anyhow::bail!("leaf saw stream error: {e:?}"),
other => anyhow::bail!("unexpected message mid-delivery: {other:?}"),
}
}
conn.close(0u32.into(), b"done");
let plaintext = {
use bao_tree::BaoTree;
use bao_tree::io::BaoContentItem;
use bao_tree::io::sync::DecodeResponseIter;
let tree = BaoTree::new(total, decdn_cache::range_pull::IROH_BLOCK_SIZE);
let reader = std::io::Cursor::new(&buf[..]);
let mut out = Vec::new();
for item in
DecodeResponseIter::new(hash.into(), tree, reader, aligned.chunk_ranges().as_ref())
{
match item.map_err(|e| anyhow::anyhow!("bao decode: {e}"))? {
BaoContentItem::Leaf(leaf) => out.extend_from_slice(&leaf.data),
BaoContentItem::Parent(_) => {}
}
}
out
};
let lead = usize::try_from(byte_offset.saturating_sub(aligned.fetch_start()))
.map_err(|e| anyhow::anyhow!("lead: {e}"))?;
let want = if byte_len == 0 {
plaintext.len().saturating_sub(lead)
} else {
let end = byte_offset.saturating_add(byte_len).min(total);
usize::try_from(end.saturating_sub(byte_offset)).map_err(|e| anyhow::anyhow!("len: {e}"))?
};
let end = lead.saturating_add(want);
plaintext
.get(lead..end)
.map(<[u8]>::to_vec)
.ok_or_else(|| anyhow::anyhow!("decoded range shorter than requested span"))
}
async fn read_client(recv: &mut iroh::endpoint::RecvStream) -> Result<ClientMessage> {
let frame = read_frame(recv)
.await
.map_err(|e| anyhow::anyhow!("leaf read frame: {e}"))?;
let (msg, _) =
decode_message::<ClientMessage>(&frame).map_err(|e| anyhow::anyhow!("leaf decode: {e}"))?;
Ok(msg)
}
async fn read_client_response(
recv: &mut iroh::endpoint::RecvStream,
) -> Result<(StreamResponse, StreamResponseExt)> {
let frame = read_frame(recv)
.await
.map_err(|e| anyhow::anyhow!("leaf read frame: {e}"))?;
let (msg, tail) =
decode_message::<ClientMessage>(&frame).map_err(|e| anyhow::anyhow!("leaf decode: {e}"))?;
let ClientMessage::StreamResponse(resp) = msg else {
anyhow::bail!("expected StreamResponse, got {msg:?}");
};
let ext = decdn_protocol::parse_stream_response_ext(tail)
.map_err(|e| anyhow::anyhow!("leaf decode ext: {e}"))?;
Ok((resp, ext))
}
async fn write_client(send: &mut iroh::endpoint::SendStream, msg: &ClientMessage) -> Result<()> {
let payload = encode_message(msg).map_err(|e| anyhow::anyhow!("leaf encode: {e}"))?;
write_frame(send, &payload)
.await
.map_err(|e| anyhow::anyhow!("leaf write: {e}"))?;
Ok(())
}
fn decode_bao_whole(hash: Hash, total: u64, wire: &[u8]) -> Option<Vec<u8>> {
use bao_tree::io::BaoContentItem;
use bao_tree::io::sync::DecodeResponseIter;
use bao_tree::{BaoTree, ChunkRanges};
let tree = BaoTree::new(total, decdn_cache::range_pull::IROH_BLOCK_SIZE);
let ranges = ChunkRanges::all();
let reader = std::io::Cursor::new(wire);
let mut plaintext = Vec::with_capacity(usize::try_from(total).ok()?);
for item in DecodeResponseIter::new(hash.into(), tree, reader, ranges.as_ref()) {
match item.ok()? {
BaoContentItem::Leaf(leaf) => plaintext.extend_from_slice(&leaf.data),
BaoContentItem::Parent(_) => {}
}
}
Some(plaintext)
}
#[allow(clippy::too_many_arguments)]
async fn leaf_paced_pull(
leaf_ep: &iroh::Endpoint,
target: EndpointAddr,
leaf_node_id: B256,
leaf_eth: &Arc<PrivateKeySigner>,
provider: Address,
pool_id: B256,
hash: Hash,
rate: u64,
drop_after_acks: Option<u64>,
) -> Result<LeafOutcome> {
let mode = drop_after_acks.map_or(LeafMode::PayAll, LeafMode::DropAfter);
leaf_paced_pull_mode(
leaf_ep,
target,
leaf_node_id,
leaf_eth,
provider,
pool_id,
hash,
rate,
mode,
0,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn leaf_paced_pull_ranged(
leaf_ep: &iroh::Endpoint,
target: EndpointAddr,
leaf_node_id: B256,
leaf_eth: &Arc<PrivateKeySigner>,
provider: Address,
pool_id: B256,
hash: Hash,
rate: u64,
byte_len: u64,
) -> Result<LeafOutcome> {
leaf_paced_pull_mode(
leaf_ep,
target,
leaf_node_id,
leaf_eth,
provider,
pool_id,
hash,
rate,
LeafMode::PayAll,
byte_len,
)
.await
}
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
async fn leaf_paced_pull_mode(
leaf_ep: &iroh::Endpoint,
target: EndpointAddr,
leaf_node_id: B256,
leaf_eth: &Arc<PrivateKeySigner>,
provider: Address,
pool_id: B256,
hash: Hash,
rate: u64,
mode: LeafMode,
byte_len: u64,
) -> Result<LeafOutcome> {
use alloy::signers::SignerSync;
let conn = leaf_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("leaf connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("leaf open_bi: {e}"))?;
let binding_hash = binding_signing_hash(leaf_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = leaf_eth.sign_hash_sync(&binding_hash)?.as_bytes().to_vec();
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: leaf_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset: 0,
byte_len,
timestamp_us: 0x9001,
};
let payload =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame(&mut send, &payload)
.await
.map_err(|e| anyhow::anyhow!("write req: {e}"))?;
let (resp, resp_ext) = read_client_response(&mut recv).await?;
anyhow::ensure!(resp.body.ok, "delivery refused: {:?}", resp_ext.error);
let total = resp.body.total_bytes;
let expected_wire =
decdn_cache::range_pull::bao_encoded_size(total, &bao_tree::ChunkRanges::all());
let interval_bytes = CHUNK_BYTES;
let mut buf = BytesMut::new();
let mut cumulative: u64 = 0;
let mut unvouchered: u64 = 0;
let mut acks: u64 = 0;
let mut paid_wire: u64 = 0;
let mut hold_until: Option<tokio::time::Instant> = None;
loop {
let msg = match hold_until {
None => read_client(&mut recv).await?,
Some(deadline) => match tokio::time::timeout_at(deadline, read_client(&mut recv)).await
{
Ok(Err(e)) => anyhow::bail!("leaf read failed during the unpaid hold: {e}"),
Ok(Ok(ClientMessage::StreamError(e))) => {
anyhow::bail!("B ended the stream during the unpaid hold: {e:?}")
}
Err(_) => {
conn.close(0u32.into(), b"leaf-hold");
return Ok(LeafOutcome {
received: cumulative,
acks,
paid_wire,
completed: false,
hash_ok: false,
});
}
Ok(Ok(msg)) => msg,
},
};
match msg {
ClientMessage::ChunkData(chunk) => {
buf.extend_from_slice(chunk.bytes());
let len = chunk.bytes().len() as u64;
cumulative = cumulative.saturating_add(len);
unvouchered = unvouchered.saturating_add(len);
let boundary = unvouchered >= interval_bytes && interval_bytes > 0;
let closing = cumulative >= expected_wire && unvouchered > 0;
if (boundary || closing) && hold_until.is_none() {
acks += 1;
let amount = U256::from(cumulative)
.saturating_mul(U256::from(rate))
.div_ceil(U256::from(MB_BYTES));
let signed = Voucher {
pool_id,
signer: leaf_eth.address(),
provider,
amount,
bytes_delivered: U256::from(cumulative),
chain_root: B256::ZERO,
chunk_price: U256::ZERO,
}
.sign(leaf_eth.as_ref(), &voucher_dom())
.map_err(|e| anyhow::anyhow!("sign voucher: {e}"))?;
write_client(
&mut send,
&ClientMessage::Voucher(signed_to_wire_voucher(&signed)?),
)
.await?;
unvouchered = 0;
paid_wire = cumulative;
match mode {
LeafMode::DropAfter(n) if n == acks => {
conn.close(0u32.into(), b"leaf-drop");
return Ok(LeafOutcome {
received: cumulative,
acks,
paid_wire,
completed: false,
hash_ok: false,
});
}
LeafMode::StopPayingAfter { acks: n, hold } if n == acks => {
hold_until = Some(tokio::time::Instant::now() + hold);
}
_ => {}
}
}
}
ClientMessage::StreamEnd => break,
ClientMessage::StreamError(e) => anyhow::bail!("leaf saw stream error: {e:?}"),
other => anyhow::bail!("unexpected message mid-delivery: {other:?}"),
}
}
conn.close(0u32.into(), b"done");
let decoded = decode_bao_whole(hash, total, &buf);
Ok(LeafOutcome {
received: decoded.as_ref().map_or(cumulative, |p| p.len() as u64),
acks,
paid_wire,
completed: true,
hash_ok: decoded.is_some_and(|p| Hash::new(&p) == hash),
})
}
#[allow(clippy::too_many_arguments)]
async fn build_node_b(
a_id: iroh::PublicKey,
a_addr: std::net::SocketAddr,
a_eth_addr: Address,
hash: Hash,
ab_pool_id: B256,
b_buyer: &Arc<PrivateKeySigner>,
leaf_pool_id: B256,
leaf_eth_addr: Address,
leaf_deposit: U256,
max_blob_size_bytes: u64,
) -> Result<(
Arc<decdn_node::handlers::client::ClientHandler>,
EndpointAddr,
iroh::Endpoint,
Arc<Mutex<Vec<ProgressEntry>>>,
decdn_cache::CacheEngine,
Arc<Metrics>,
Arc<LocalReputation>,
Address,
)> {
build_node_b_with_leaves(
a_id,
a_addr,
a_eth_addr,
hash,
ab_pool_id,
b_buyer,
&[(leaf_pool_id, leaf_eth_addr, leaf_eth_addr, leaf_deposit)],
max_blob_size_bytes,
64,
None,
DEFAULT_TEST_PULL_DEADLINES,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn build_node_b_with_leaves(
a_id: iroh::PublicKey,
a_addr: std::net::SocketAddr,
a_eth_addr: Address,
hash: Hash,
ab_pool_id: B256,
b_buyer: &Arc<PrivateKeySigner>,
leaves: &[(B256, Address, Address, U256)],
max_blob_size_bytes: u64,
engine_max_blob_mb: u64,
content_deny: Option<Arc<decdn_node::content_deny::ContentDenylist>>,
node_pull_deadlines: (Duration, Duration),
) -> Result<(
Arc<decdn_node::handlers::client::ClientHandler>,
EndpointAddr,
iroh::Endpoint,
Arc<Mutex<Vec<ProgressEntry>>>,
decdn_cache::CacheEngine,
Arc<Metrics>,
Arc<LocalReputation>,
Address,
)> {
build_node_b_with_store(
a_id,
a_addr,
a_eth_addr,
hash,
ab_pool_id,
b_buyer,
leaves,
max_blob_size_bytes,
engine_max_blob_mb,
content_deny,
node_pull_deadlines,
|store| Arc::new(store) as Arc<dyn PoolStateStore>,
)
.await
}
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
async fn build_node_b_with_store(
a_id: iroh::PublicKey,
a_addr: std::net::SocketAddr,
a_eth_addr: Address,
hash: Hash,
ab_pool_id: B256,
b_buyer: &Arc<PrivateKeySigner>,
leaves: &[(B256, Address, Address, U256)],
max_blob_size_bytes: u64,
engine_max_blob_mb: u64,
content_deny: Option<Arc<decdn_node::content_deny::ContentDenylist>>,
node_pull_deadlines: (Duration, Duration),
wrap_store: impl FnOnce(MemoryPoolStateStore) -> Arc<dyn PoolStateStore>,
) -> Result<(
Arc<decdn_node::handlers::client::ClientHandler>,
EndpointAddr,
iroh::Endpoint,
Arc<Mutex<Vec<ProgressEntry>>>,
decdn_cache::CacheEngine,
Arc<Metrics>,
Arc<LocalReputation>,
Address,
)> {
let b_sk = fresh_key();
let b_id = b_sk.public();
let b_eth = Arc::new(PrivateKeySigner::random());
let (ep_b, addr_b) = local_endpoint(b_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(a_addr),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) = one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth_addr);
let (origin, _engine, recorded, _engine_tmp) = provisioned_origin_with_deadlines(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
ab_pool_id,
b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
node_pull_deadlines,
max_blob_size_bytes,
)
.await;
let cache_tmp = tempfile::tempdir()?;
let cache_b =
decdn_cache::CacheEngine::open(cache_tmp.path(), vec![], engine_max_blob_mb).await?;
let cache_handle = cache_b.clone();
std::mem::forget(cache_tmp);
let store_b = MemoryPoolStateStore::new();
let mut pool_status_map: HashMap<B256, decdn_node::pool_view::PoolStatus> = HashMap::new();
for (leaf_pool_id, leaf_funder, leaf_voucher_signer, leaf_deposit) in leaves {
store_b.record(&LaneState::hydrate(
*leaf_pool_id,
*leaf_voucher_signer,
b_eth.address(),
*leaf_deposit,
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
pool_status_map.insert(
*leaf_pool_id,
decdn_node::pool_view::PoolStatus {
owner: *leaf_funder,
remaining: *leaf_deposit,
lifecycle: decdn_node::pool_view::Lifecycle::Open,
},
);
}
let pool_view = Arc::new(StubPoolView {
status: pool_status_map,
}) as Arc<dyn decdn_node::pool_view::PoolView>;
let limiter = permissive_limiter(&b_metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_b = build_handler_full_configured(
b_id,
&b_eth,
&b_metrics,
limiter,
cache_b,
wrap_store(store_b),
RATE,
&domains,
16,
|deps| {
deps.pull_through = Some(Duration::from_secs(20));
deps.pull_through_origin = Some(Arc::new(origin));
deps.pool_view = Some(pool_view);
if let Some(deny) = content_deny {
deps.content_deny = deny;
}
},
)?;
let target = EndpointAddr::new(b_id).with_ip_addr(addr_b);
Ok((
handler_b,
target,
ep_b,
recorded,
cache_handle,
b_metrics,
local_rep,
b_eth.address(),
))
}
async fn spawn_node_a(
payload: &[u8],
ab_pool_id: B256,
b_buyer_addr: Address,
) -> Result<(
iroh::PublicKey,
std::net::SocketAddr,
Arc<PrivateKeySigner>,
iroh::Endpoint,
tokio::task::JoinHandle<()>,
)> {
let (a_id, addr_a, a_eth, ep_a, task_a, _metrics) =
spawn_node_a_metered(payload, ab_pool_id, b_buyer_addr).await?;
Ok((a_id, addr_a, a_eth, ep_a, task_a))
}
async fn spawn_node_a_metered(
payload: &[u8],
ab_pool_id: B256,
b_buyer_addr: Address,
) -> Result<(
iroh::PublicKey,
std::net::SocketAddr,
Arc<PrivateKeySigner>,
iroh::Endpoint,
tokio::task::JoinHandle<()>,
Arc<Metrics>,
)> {
let hash = Hash::new(payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let (cache_a, hash_a, tmp_a) = cache_with_blob(payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
std::mem::forget(tmp_a);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
ab_pool_id,
b_buyer_addr,
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
Ok((a_id, addr_a, a_eth, ep_a, task_a, metrics))
}
async fn await_run_outcome(
spans: &support::SpanCapture,
hash: Hash,
outcome: &str,
a_id: iroh::PublicKey,
pool_id: B256,
) -> Result<()> {
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
loop {
let runs = spans.matching("upstream_stream", "hash", &hash.to_string());
if runs.iter().any(|s| {
s.parent == Some("serve_miss_pull")
&& s.fields.get("outcome").map(String::as_str) == Some(outcome)
&& s.fields.get("peer") == Some(&a_id.to_string())
&& s.fields.get("pool_id") == Some(&pool_id.to_string())
}) {
return Ok(());
}
anyhow::ensure!(
tokio::time::Instant::now() < deadline,
"no `{outcome}` upstream_stream run: {runs:?}"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
async fn assert_miss_serve_waits(
spans: &support::SpanCapture,
peer: &str,
hash: Hash,
payload_len: u64,
) -> Result<()> {
let key = hash.to_string();
let serve = || {
spans
.matching("serve_stream", "peer", peer)
.into_iter()
.find(|s| s.fields.get("hash") == Some(&key) && s.fields.contains_key("store_wait_ns"))
};
let deadline = std::time::Instant::now() + Duration::from_secs(10);
while serve().is_none() && std::time::Instant::now() < deadline {
tokio::time::sleep(Duration::from_millis(20)).await;
}
let span = serve().ok_or_else(|| anyhow::anyhow!("no serve_stream span to {peer}"))?;
support::assert_serve_waits_recorded(&span, payload_len)?;
anyhow::ensure!(
span.fields
.get("store_wait_ns")
.and_then(|v| v.parse::<u64>().ok())
.is_some_and(|ns| ns > 0),
"the miss waits on the pull: {span:?}"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_serves_and_caches_full_blob() -> Result<()> {
let spans = support::capture_spans();
let window = decdn_client::PULL_WINDOW_FLOOR;
let payload_len = usize::try_from(window.saturating_mul(3) / 2).unwrap_or(usize::MAX);
let payload = vec![0xCDu8; payload_len];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload_len).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA1);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x1F);
let (handler_b, b_target, ep_b, recorded, cache_b, b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await?;
anyhow::ensure!(outcome.completed, "leaf delivery did not complete");
anyhow::ensure!(outcome.hash_ok, "leaf received bytes failed the hash check");
anyhow::ensure!(
outcome.received == total_bytes,
"leaf received {} of {total_bytes} bytes",
outcome.received
);
anyhow::ensure!(
counter_value(&b_metrics, "node_pull_through_window_paused_total")? >= 1,
"the window pause/resume path must have engaged for a blob larger than the window"
);
anyhow::ensure!(
cache_b.has(hash).await?,
"B must promote the teed blob on a complete delivery"
);
let leg1_wire =
u64::try_from(honest_bao_wire_range(&payload, 0, window)?.len()).unwrap_or(u64::MAX);
let leg2_wire =
u64::try_from(honest_bao_wire_range(&payload, window, 0)?.len()).unwrap_or(u64::MAX);
let expected_wire = leg1_wire.saturating_add(leg2_wire);
let per_leg_amount = |wire: u64| -> U256 {
let mut amount = U256::ZERO;
let mut remaining = wire;
while remaining > 0 {
let chunk = remaining.min(CHUNK_BYTES);
amount += min_payment(chunk, RATE);
remaining -= chunk;
}
amount
};
let expected_amount = per_leg_amount(leg1_wire) + per_leg_amount(leg2_wire);
anyhow::ensure!(
progress_log(&recorded)?
== vec![(a_eth.address(), U256::from(expected_wire), expected_amount)],
"expected B's upstream watermark at the full blob ({expected_wire} wire bytes, \
{expected_amount} paid), got {:?}",
progress_log(&recorded)?
);
assert_relay_counted(&b_metrics, u64::try_from(payload.len())?, 2).await?;
assert_miss_serve_waits(&spans, &leaf_ep.id().to_string(), hash, total_bytes).await?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
await_run_outcome(&spans, hash, "filled", a_id, ab_pool_id).await?;
let opens = spans.matching("open_progressive_pull", "hash", &hash.to_string());
let leg_parent = |offset: &str| {
opens
.iter()
.find(|s| s.fields.get("byte_offset").map(String::as_str) == Some(offset))
.and_then(|s| s.parent)
};
anyhow::ensure!(
leg_parent("0") == Some("serve_stream")
&& leg_parent(&window.to_string()) == Some("upstream_stream"),
"the handshake leg nests under serve_stream, the later leg under upstream_stream: \
{opens:?}"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_handshake_is_the_first_pull_leg() -> Result<()> {
let window = decdn_client::PULL_WINDOW_FLOOR;
let payload_len = usize::try_from(window.saturating_mul(3) / 2).unwrap_or(usize::MAX);
let payload = vec![0x5Au8; payload_len];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload_len).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA1);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a, a_metrics) =
spawn_node_a_metered(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x1F);
let (handler_b, b_target, ep_b, _recorded, _cache_b, _b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await?;
anyhow::ensure!(
outcome.completed && outcome.hash_ok && outcome.received == total_bytes,
"leaf delivery must complete byte-exact: {outcome:?}"
);
let served = counter_value(&a_metrics, "serve_cache_hit_total")?;
anyhow::ensure!(
served == 2,
"A must serve only the two paid legs, the first of them the handshake's: served {served}"
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_completes_a_blob_past_the_ramp_floor() -> Result<()> {
let payload_len: usize = 8 * 1024 * 1024;
let mut payload = vec![0u8; payload_len];
let mut x: u32 = 0x2545_f491;
for b in &mut payload {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
*b = x.to_le_bytes().first().copied().unwrap_or(0);
}
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload_len).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA3);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x3F);
let (handler_b, b_target, ep_b, _recorded, cache_b, b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = tokio::time::timeout(
Duration::from_mins(1),
leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
),
)
.await
.map_err(|_| anyhow::anyhow!("the fused serve-miss delivery stalled"))??;
anyhow::ensure!(outcome.completed, "leaf delivery did not complete");
anyhow::ensure!(outcome.hash_ok, "leaf received bytes failed the hash check");
anyhow::ensure!(
outcome.received == total_bytes,
"leaf received {} of {total_bytes} bytes",
outcome.received
);
anyhow::ensure!(
counter_value(&b_metrics, "node_pull_through_window_paused_total")? >= 1,
"the pull must have paused on its window for a blob this far past the floor"
);
anyhow::ensure!(
cache_b.has(hash).await?,
"B must promote the teed blob on a complete delivery"
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_funder_blacklisted_mid_stream_cuts_off_a_delegated_delivery()
-> Result<()> {
let payload = vec![0x6Bu8; 12 * 1024 * 1024];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA7);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_funder = Arc::new(PrivateKeySigner::random());
let leaf_delegate = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x2F);
let deny = Arc::new(decdn_node::content_deny::ContentDenylist::empty());
let (handler_b, b_target, ep_b, _recorded, _cache_b, b_metrics, _local_rep, b_operator) =
build_node_b_with_leaves(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
&[(
leaf_pool_id,
leaf_funder.address(),
leaf_delegate.address(),
U256::from(DEPOSIT_MICRO_USDC),
)],
0,
64,
Some(Arc::clone(&deny)),
DEFAULT_TEST_PULL_DEADLINES,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let pull_ep = leaf_ep.clone();
let leaf_signer = Arc::clone(&leaf_delegate);
let leaf = tokio::spawn(async move {
leaf_paced_pull(
&pull_ep,
b_target,
leaf_node_id,
&leaf_signer,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await
});
let deadline = std::time::Instant::now() + Duration::from_secs(30);
while counter_value(&b_metrics, "node_pull_through_window_paused_total")? == 0 {
anyhow::ensure!(
std::time::Instant::now() < deadline,
"the window pull-through loop never engaged; the test never reached the \
mid-stream re-check"
);
tokio::time::sleep(Duration::from_millis(1)).await;
}
deny.apply_chain_origin(leaf_funder.address(), true);
let outcome = tokio::time::timeout(Duration::from_secs(45), leaf)
.await
.map_err(|_| anyhow::anyhow!("the leaf pull never returned"))??;
anyhow::ensure!(
outcome.is_err(),
"delivery completed for a blacklisted funder — the window loop's mid-stream \
re-check has been re-keyed onto the voucher signer: {outcome:?}"
);
support::assert_inbound_failures_attributed(&b_metrics, 1).await?;
assert_counter(&b_metrics, "serve_stream_terminated_takedown_total", 1)?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_local_fault_refuses_internal_error_not_not_found() -> Result<()> {
let payload = vec![0xC1u8; 64 * 1024];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA1);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x1F);
let (handler_b, b_target, ep_b, _recorded, _cache_b, b_metrics, _local_rep, b_operator) =
build_node_b_with_leaves(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
&[(
leaf_pool_id,
leaf_eth.address(),
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
)],
0,
64,
None,
(Duration::from_secs(20), Duration::ZERO),
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let refusal = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await
.err()
.ok_or_else(|| anyhow::anyhow!("B cannot pay for the upstream pull, so it cannot serve"))?;
let refusal = refusal.to_string();
anyhow::ensure!(
refusal.contains("Declined"),
"B's own broken deadline config must be reported as B declining, got: {refusal}"
);
anyhow::ensure!(
!refusal.contains("NotFound"),
"the blob is on A, one hop away — signing the leaf a `NotFound` is a false claim \
about the content, which is the whole of #1560, got: {refusal}"
);
assert_counter(&b_metrics, "serve_stream_rejected_internal_error_total", 1)?;
assert_counter(&b_metrics, "serve_stream_rejected_cache_miss_total", 0)?;
assert_counter(&b_metrics, "node_pull_local_fault_total", 1)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 0)?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_honest_upstream_miss_still_refuses_not_found() -> Result<()> {
let payload = vec![0xC2u8; 64 * 1024];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA2);
let b_buyer = Arc::new(PrivateKeySigner::random());
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, a_addr) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_refusing_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
StreamError::NotFound,
);
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x2F);
let (handler_b, b_target, ep_b, _recorded, _cache_b, b_metrics, _local_rep, b_operator) =
build_node_b_with_leaves(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
&[(
leaf_pool_id,
leaf_eth.address(),
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
)],
0,
64,
None,
DEFAULT_TEST_PULL_DEADLINES,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let refusal = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await
.err()
.ok_or_else(|| anyhow::anyhow!("no upstream holds the blob, so B cannot serve it"))?;
let refusal = refusal.to_string();
anyhow::ensure!(
refusal.contains("NotFound"),
"an honest network-wide miss is exactly what a wire `NotFound` is for, got: {refusal}"
);
anyhow::ensure!(
!refusal.contains("Declined"),
"B is healthy — declining an ordinary miss would steer clients off a working \
node on every cache miss, got: {refusal}"
);
assert_counter(&b_metrics, "serve_stream_rejected_cache_miss_total", 1)?;
assert_counter(&b_metrics, "serve_stream_rejected_internal_error_total", 0)?;
assert_counter(&b_metrics, "node_pull_local_fault_total", 0)?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_serves_and_caches_empty_blob() -> Result<()> {
let payload: Vec<u8> = Vec::new();
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA1);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x1F);
let (handler_b, b_target, ep_b, recorded, cache_b, _b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await?;
anyhow::ensure!(outcome.completed, "leaf delivery did not complete");
anyhow::ensure!(
outcome.hash_ok,
"leaf received bytes failed the empty-root check"
);
anyhow::ensure!(
outcome.received == 0,
"leaf received {} bytes, want 0",
outcome.received
);
anyhow::ensure!(
cache_b.has(hash).await?,
"B must promote the teed empty blob"
);
anyhow::ensure!(
progress_log(&recorded)?.is_empty(),
"no upstream voucher for a 0-byte blob, got {:?}",
progress_log(&recorded)?
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn window_pull_through_concurrent_same_hash_single_upstream_pull() -> Result<()> {
let payload = vec![0xC0u8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA8);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a, received, release) =
spawn_gated_node_a(&payload).await?;
let leaf1_eth = Arc::new(PrivateKeySigner::random());
let leaf2_eth = Arc::new(PrivateKeySigner::random());
let leaf1_pool_id = B256::repeat_byte(0x81);
let leaf2_pool_id = B256::repeat_byte(0x82);
let (handler_b, b_target, ep_b, recorded, cache_b, b_metrics, _local_rep, b_operator) =
build_node_b_with_leaves(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
&[
(
leaf1_pool_id,
leaf1_eth.address(),
leaf1_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
),
(
leaf2_pool_id,
leaf2_eth.address(),
leaf2_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
),
],
0,
64,
None,
DEFAULT_TEST_PULL_DEADLINES,
)
.await?;
let task_b = spawn_server_concurrent(ep_b.clone(), handler_b);
let leaf1_sk = fresh_key();
let leaf1_node_id = B256::from(*leaf1_sk.public().as_bytes());
let (leaf1_ep, _) = local_endpoint(leaf1_sk, vec![]).await?;
let leaf1_target = b_target.clone();
let leaf1_eth_c = Arc::clone(&leaf1_eth);
let leaf1_task = tokio::spawn(async move {
leaf_paced_pull(
&leaf1_ep,
leaf1_target,
leaf1_node_id,
&leaf1_eth_c,
b_operator,
leaf1_pool_id,
hash,
RATE,
None,
)
.await
});
tokio::time::timeout(Duration::from_secs(20), received.notified())
.await
.map_err(|_| {
anyhow::anyhow!(
"gated upstream A never signaled `received` — its task likely errored \
before reading B's upstream StreamRequest"
)
})?;
tokio::time::timeout(Duration::from_secs(20), async {
while cache_b.in_flight_total(hash).is_none() {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.map_err(|_| anyhow::anyhow!("B never claimed its fill for leaf 1"))?;
let leaf2_sk = fresh_key();
let leaf2_node_id = B256::from(*leaf2_sk.public().as_bytes());
let (leaf2_ep, _) = local_endpoint(leaf2_sk, vec![]).await?;
let leaf2_target = b_target.clone();
let leaf2_eth_c = Arc::clone(&leaf2_eth);
let leaf2_task = tokio::spawn(async move {
leaf_paced_pull(
&leaf2_ep,
leaf2_target,
leaf2_node_id,
&leaf2_eth_c,
b_operator,
leaf2_pool_id,
hash,
RATE,
None,
)
.await
});
tokio::time::sleep(Duration::from_millis(250)).await;
release.notify_one();
let out1 = leaf1_task.await??;
let out2 = leaf2_task.await??;
for (label, out) in [("leaf 1", &out1), ("leaf 2", &out2)] {
anyhow::ensure!(out.completed, "{label} delivery did not complete");
anyhow::ensure!(out.hash_ok, "{label} received bytes failed the hash check");
anyhow::ensure!(
out.received == total_bytes,
"{label} received {} of {total_bytes} bytes",
out.received
);
}
let upstream = progress_log(&recorded)?;
anyhow::ensure!(
upstream.len() == 1,
"concurrent same-hash requests must open ONE upstream pull, got {upstream:?}"
);
let entry = upstream
.first()
.ok_or_else(|| anyhow::anyhow!("no upstream watermark recorded"))?;
anyhow::ensure!(
u64::try_from(entry.1).unwrap_or(u64::MAX) == total_bytes,
"the single upstream pull must cover exactly one blob, got {entry:?}"
);
anyhow::ensure!(
cache_b.has(hash).await?,
"B must promote the single coalesced fill"
);
anyhow::ensure!(
counter_value(&b_metrics, "node_pull_through_upstream_verify_failed_total")? == 0,
"an honest upstream must not trip the verify-failed counter"
);
shutdown([task_a, task_b], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines, clippy::similar_names)]
async fn concurrent_same_lane_misses_refuse_surplus() -> Result<()> {
use alloy::signers::SignerSync;
let payload = vec![0xC7u8; 4096];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA9);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a, received, release) =
spawn_gated_node_a(&payload).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x8A);
let remaining = decdn_incentive::min_payment(CHUNK_BYTES, RATE) + U256::from(2u64);
let (handler_b, b_target, ep_b, _recorded, cache_b, _b_metrics, _local_rep, b_operator) =
build_node_b_with_leaves(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
&[(
leaf_pool_id,
leaf_eth.address(),
leaf_eth.address(),
remaining,
)],
0,
64,
None,
DEFAULT_TEST_PULL_DEADLINES,
)
.await?;
let task_b = spawn_server_concurrent(ep_b.clone(), handler_b);
let leaf1_sk = fresh_key();
let leaf1_node_id = B256::from(*leaf1_sk.public().as_bytes());
let (leaf1_ep, _) = local_endpoint(leaf1_sk, vec![]).await?;
let leaf1_target = b_target.clone();
let leaf1_eth = Arc::clone(&leaf_eth);
let leaf1_task = tokio::spawn(async move {
leaf_paced_pull(
&leaf1_ep,
leaf1_target,
leaf1_node_id,
&leaf1_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await
});
tokio::time::timeout(Duration::from_secs(20), received.notified())
.await
.map_err(|_| {
anyhow::anyhow!(
"gated upstream A never signaled `received` — leaf 1 likely never \
reached the pull-through path"
)
})?;
let leaf2_sk = fresh_key();
let leaf2_node_id = B256::from(*leaf2_sk.public().as_bytes());
let (leaf2_ep, _) = local_endpoint(leaf2_sk, vec![]).await?;
let conn2 = leaf2_ep
.connect(b_target.clone(), ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("leaf2 connect: {e}"))?;
let (mut send2, mut recv2) = conn2
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("leaf2 open_bi: {e}"))?;
let binding_hash = binding_signing_hash(leaf2_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = leaf_eth.sign_hash_sync(&binding_hash)?.as_bytes().to_vec();
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: leaf_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: leaf_pool_id.into(),
byte_offset: 0,
byte_len: 0,
timestamp_us: 0x9002,
};
let payload2 =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame(&mut send2, &payload2)
.await
.map_err(|e| anyhow::anyhow!("write req: {e}"))?;
let frame2 = read_frame(&mut recv2)
.await
.map_err(|e| anyhow::anyhow!("read resp: {e}"))?;
let (msg2, rest2) = decode_message::<ClientMessage>(&frame2)
.map_err(|e| anyhow::anyhow!("decode resp: {e}"))?;
let ClientMessage::StreamResponse(resp2) = msg2 else {
anyhow::bail!("leaf2: expected StreamResponse, got {msg2:?}");
};
let resp2_ext = decdn_protocol::parse_stream_response_ext(rest2)
.map_err(|e| anyhow::anyhow!("decode resp ext: {e}"))?;
anyhow::ensure!(
!resp2.body.ok,
"concurrent same-lane MISS must be refused while budget covers only one floor"
);
anyhow::ensure!(
matches!(resp2_ext.error, Some(StreamError::Unfunded)),
"expected the Unfunded wire code for the pool-ceiling refusal, \
got {:?}",
resp2_ext.error
);
conn2.close(0u32.into(), b"refused");
release.notify_one();
let out1 = leaf1_task.await??;
anyhow::ensure!(out1.completed, "leaf 1 delivery did not complete");
anyhow::ensure!(out1.hash_ok, "leaf 1 received bytes failed the hash check");
anyhow::ensure!(
cache_b.has(hash).await?,
"leaf 1's fill must promote the blob"
);
shutdown([task_a, task_b], [&leaf2_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn window_pull_through_bounded_unaligned_range_pulls_only_the_span() -> Result<()> {
let payload: Vec<u8> = (0..PAYLOAD_LEN)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect();
let hash = Hash::new(&payload);
let total = u64::try_from(PAYLOAD_LEN)?;
let ab_pool_id = B256::repeat_byte(0xA9);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x8F);
let (handler_b, b_target, ep_b, recorded, cache_b, _b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let (req_off, req_len) = (20 * 1024u64, 40 * 1024u64);
let aligned = decdn_cache::range_pull::align_range(req_off, req_len, total)
.map_err(|e| anyhow::anyhow!("align: {e}"))?;
let got = leaf_ranged_paid_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
req_off,
req_len,
RATE,
)
.await?;
let want = payload
.get(usize::try_from(req_off)?..usize::try_from(req_off + req_len)?)
.ok_or_else(|| anyhow::anyhow!("requested range out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"unaligned bounded range through the peer path must be byte-exact"
);
let span_wire = decdn_cache::range_pull::bao_encoded_size(total, aligned.chunk_ranges());
let log = progress_log(&recorded)?;
let last = log
.last()
.ok_or_else(|| anyhow::anyhow!("B must have pulled upstream"))?;
anyhow::ensure!(
last.1 == U256::from(span_wire),
"B's upstream wire must be the aligned span's bao size ({span_wire}), got {}",
last.1
);
anyhow::ensure!(!cache_b.has(hash).await?, "B holds a partial, not the blob");
anyhow::ensure!(
cache_b
.missing_ranges(hash, req_off, req_len, total)
.await?
.is_empty(),
"the requested span must be present in B's cache"
);
anyhow::ensure!(
!cache_b
.missing_ranges(hash, 0, aligned.fetch_start(), total)
.await?
.is_empty(),
"the head before the aligned span must never have been pulled"
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn window_pull_through_bounded_miss_seeds_only_the_held_ranges_inside_it() -> Result<()> {
const G: u64 = 16 * 1024;
let payload: Vec<u8> = (0..PAYLOAD_LEN)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect();
let hash = Hash::new(&payload);
let total = u64::try_from(PAYLOAD_LEN)?;
let outboard = bao_tree::io::outboard::PreOrderMemOutboard::create(
&payload,
decdn_cache::range_pull::IROH_BLOCK_SIZE,
);
let outboard = bytes::Bytes::from(outboard.data);
let ab_pool_id = B256::repeat_byte(0xAA);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x90);
let (handler_b, b_target, ep_b, recorded, cache_b, _b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
for (off, len) in [(0, G), (4 * G, 4 * G), (total - G, G)] {
let aligned = decdn_cache::range_pull::align_range(off, len, total)?;
let data = payload
.get(usize::try_from(aligned.fetch_start())?..usize::try_from(aligned.fetch_end())?)
.ok_or_else(|| anyhow::anyhow!("held span out of bounds"))?;
let bao = decdn_cache::range_pull::encode_verified_range(
*hash.as_bytes(),
&aligned,
data,
outboard.clone(),
)?;
cache_b
.admit_bao(hash, aligned.chunk_ranges().clone(), bao)
.await?;
}
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let (req_off, req_len) = (3 * G + 100, 7 * G - 150);
let aligned = decdn_cache::range_pull::align_range(req_off, req_len, total)
.map_err(|e| anyhow::anyhow!("align: {e}"))?;
let got = tokio::time::timeout(
Duration::from_secs(30),
leaf_ranged_paid_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
req_off,
req_len,
RATE,
),
)
.await
.map_err(|_| anyhow::anyhow!("the bounded miss over held ranges hung"))??;
let want = payload
.get(usize::try_from(req_off)?..usize::try_from(req_off + req_len)?)
.ok_or_else(|| anyhow::anyhow!("requested range out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"bounded peer miss over held ranges must be byte-exact"
);
let span_wire = decdn_cache::range_pull::bao_encoded_size(total, aligned.chunk_ranges());
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
let last = loop {
if let Some(last) = progress_log(&recorded)?.last().copied() {
break last;
}
anyhow::ensure!(
tokio::time::Instant::now() < deadline,
"B must have pulled the gaps upstream"
);
tokio::time::sleep(Duration::from_millis(20)).await;
};
anyhow::ensure!(
last.1 < U256::from(span_wire),
"B's upstream wire ({}) must stay below the aligned span's bao size ({span_wire})",
last.1
);
anyhow::ensure!(
cache_b
.missing_ranges(hash, req_off, req_len, total)
.await?
.is_empty(),
"the requested span must be present in B's cache"
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn window_pull_through_resumed_offset_is_served_by_the_fused_path() -> Result<()> {
let payload = vec![0x7Eu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA7);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x7F);
let (handler_b, b_target, ep_b, _recorded, _cache_b, _b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let got = leaf_ranged_paid_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
MB_BYTES,
0,
RATE,
)
.await?;
let want = payload
.get(usize::try_from(MB_BYTES)?..)
.ok_or_else(|| anyhow::anyhow!("tail out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"the fused path must serve the requested tail byte-exact"
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_end_past_the_blob_is_served_by_the_fused_path_clamped() -> Result<()> {
let payload = vec![0x7Eu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total = u64::try_from(PAYLOAD_LEN)?;
let ab_pool_id = B256::repeat_byte(0xA6);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x76);
let (handler_b, b_target, ep_b, _recorded, _cache_b, b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let req_off = MB_BYTES;
let got = leaf_ranged_paid_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
req_off,
total,
RATE,
)
.await?;
let want = payload
.get(usize::try_from(req_off)?..)
.ok_or_else(|| anyhow::anyhow!("tail out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"an end past the blob must serve clamped to the blob's end through the \
fused window path, byte-exact"
);
assert_relay_counted(&b_metrics, total - req_off, 1).await?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_drop_after_fill_bounds_upstream_spend() -> Result<()> {
let payload = vec![0x4Du8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA2);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x2F);
let (handler_b, b_target, ep_b, recorded, _cache_b, _b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
Some(1),
)
.await?;
anyhow::ensure!(
!outcome.completed,
"leaf was supposed to drop, not complete"
);
anyhow::ensure!(
outcome.acks == 1,
"leaf should have acked exactly one voucher"
);
tokio::time::sleep(Duration::from_millis(300)).await;
let log = progress_log(&recorded)?;
let upstream_bytes: u64 = log
.last()
.map_or(0, |(_, bytes, _)| u64::try_from(*bytes).unwrap_or(u64::MAX));
let paid = outcome.received;
let one_window = MB_BYTES;
let group = decdn_cache::CHUNK_GROUP_BYTES;
anyhow::ensure!(
upstream_bytes.saturating_sub(paid) <= one_window + group,
"B's UNRECOUPED upstream lead ({upstream_bytes} pulled - {paid} paid) must be bounded to \
~one window ({one_window}), not run open-ended against the {total_bytes}-byte blob"
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn window_pull_through_connected_nonpaying_leaf_past_ramp_bounds_upstream_spend() -> Result<()>
{
let spans = support::capture_spans();
let payload_len = 8 * CHUNK_BYTES;
let payload = vec![0x5Au8; usize::try_from(payload_len)?];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA9);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x9A);
let (handler_b, b_target, ep_b, recorded, cache_b, b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = tokio::time::timeout(
Duration::from_mins(1),
leaf_paced_pull_mode(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
LeafMode::StopPayingAfter {
acks: 3,
hold: Duration::from_secs(2),
},
0,
),
)
.await
.map_err(|_| anyhow::anyhow!("leaf pull did not finish within a minute"))??;
anyhow::ensure!(
!outcome.completed,
"leaf stopped paying, so B must not deliver the whole blob"
);
anyhow::ensure!(
outcome.acks == 3,
"leaf should have paid exactly three vouchers, got {}",
outcome.acks
);
let paid = outcome.paid_wire;
let group = decdn_cache::CHUNK_GROUP_BYTES;
let serve_window = decdn_incentive::ramped_credit_window(
decdn_common::config::DEFAULT_CREDIT_RAMP_DIVISOR,
CHUNK_BYTES,
decdn_common::config::DEFAULT_CREDIT_MAX,
paid,
);
anyhow::ensure!(
outcome.received + group >= paid + serve_window,
"B must fill its serve credit window ({serve_window}) past the last voucher \
during the hold ({} received, {paid} paid)",
outcome.received
);
anyhow::ensure!(
counter_value(&b_metrics, "node_pull_through_window_paused_total")? >= 1,
"B's pull leg must have paused on its window"
);
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
let upstream_wire = loop {
if let Some(&(_, bytes, _)) = progress_log(&recorded)?.last() {
break u64::try_from(bytes)
.map_err(|_| anyhow::anyhow!("upstream watermark {bytes} overflows u64"))?;
}
anyhow::ensure!(
tokio::time::Instant::now() < deadline,
"B never recorded an upstream watermark: the pull leg did not settle \
(persist failures: {})",
counter_value(&b_metrics, "node_pull_progress_persist_failures_total")?
);
tokio::time::sleep(Duration::from_millis(20)).await;
};
let pull_floor = decdn_client::PULL_WINDOW_FLOOR;
let ramped = decdn_incentive::ramped_credit_window(
decdn_common::config::DEFAULT_CREDIT_RAMP_DIVISOR,
pull_floor,
decdn_common::config::DEFAULT_CREDIT_MAX,
paid,
);
anyhow::ensure!(
ramped > pull_floor,
"the pull window must have ramped past its floor ({ramped} <= {pull_floor}), \
or this test does not cover the ramp"
);
let proof_path = support::bao_wire_len(payload_len, 0, group).saturating_sub(group);
let max_opens = 32;
let bound_wire =
support::bao_wire_len(payload_len, 0, ramped + pull_floor) + max_opens * proof_path;
let lead = upstream_wire.saturating_sub(paid);
anyhow::ensure!(
lead <= bound_wire,
"B's unrecouped upstream lead ({upstream_wire} upstream - {paid} paid = {lead}) must \
stay within the ramped window ({ramped}) plus one pull-window floor ({pull_floor}) \
plus {max_opens} proof paths of {proof_path} = {bound_wire} wire bytes; leaf \
received {}",
outcome.received
);
let whole_wire = support::bao_wire_len_whole(payload_len);
anyhow::ensure!(
upstream_wire < whole_wire,
"a leaf that stopped paying must not make B pull the whole blob \
({upstream_wire} >= {whole_wire})"
);
anyhow::ensure!(
!cache_b.has(hash).await?,
"B must not cache a blob it never finished pulling"
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
await_run_outcome(&spans, hash, "cancelled", a_id, ab_pool_id).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_insufficient_deposit_refuses_before_pulling() -> Result<()> {
let payload = vec![0x9Eu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA3);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x3F);
let max_blob_size_bytes = 64 * 1024 * 1024;
let (handler_b, b_target, ep_b, recorded, cache_b, b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(5u64),
max_blob_size_bytes,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let refused = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await
.is_err();
anyhow::ensure!(
refused,
"an underfunded pool must be refused (signed NotFound), not served"
);
anyhow::ensure!(
progress_log(&recorded)?.is_empty(),
"the deposit guard must reject before any upstream spend, got {:?}",
progress_log(&recorded)?
);
anyhow::ensure!(
!cache_b.has(hash).await?,
"a deposit-refused request must not fill B's cache"
);
assert_counter(
&b_metrics,
"serve_stream_rejected_insufficient_deposit_total",
1,
)?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
async fn spawn_lying_node_a(
served_wrong: Vec<u8>,
advertised_bytes: u64,
) -> Result<(
iroh::PublicKey,
std::net::SocketAddr,
Arc<PrivateKeySigner>,
iroh::Endpoint,
tokio::task::JoinHandle<()>,
)> {
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_lying_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
served_wrong,
advertised_bytes,
RATE,
);
Ok((a_id, addr_a, a_eth, ep_a, task_a))
}
#[derive(Debug, Default, Clone, Copy)]
struct ScriptedLane {
anchor: U256,
price: U256,
index: u8,
root: B256,
credited: U256,
}
impl ScriptedLane {
fn claim(self) -> U256 {
self.anchor
.saturating_add(self.price.saturating_mul(U256::from(self.index)))
}
fn apply(&mut self, proof: &ClientMessage, stream_root: Option<B256>) -> bool {
let before = self.claim();
match proof {
ClientMessage::Voucher(v) if U256::from(v.amount) < self.anchor => {}
ClientMessage::Voucher(v) => {
let root = B256::from(v.chain_root);
if root != self.root {
self.root = root;
self.index = 0;
}
self.anchor = self.anchor.max(U256::from(v.amount));
self.price = U256::from(v.chunk_price);
}
ClientMessage::ChunkPreimage(p) => {
if stream_root == Some(self.root) {
self.index = self.index.max(p.index);
}
}
_ => return false,
}
self.claim() > before
}
fn credit(&mut self, proof: &ClientMessage, advanced: bool, served: u64) -> Credit {
if matches!(proof, ClientMessage::Voucher(v) if v.chain_root != [0u8; 32])
&& served >= CHUNK_BYTES
{
return Credit::Unpaid;
}
let due = chunk_due(self.price, served);
let headroom = self.claim().saturating_sub(self.credited);
if advanced {
self.credited = self.credited.saturating_add(due.min(headroom));
Credit::Paid
} else if headroom >= due {
self.credited = self.credited.saturating_add(due);
Credit::Paid
} else {
Credit::Refused
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Credit {
Paid,
Unpaid,
Refused,
}
async fn read_proof(recv: &mut iroh::endpoint::RecvStream) -> Result<ClientMessage> {
let frame = read_frame(recv)
.await
.map_err(|e| anyhow::anyhow!("read proof: {e}"))?;
let (msg, _) =
decode_message::<ClientMessage>(&frame).map_err(|e| anyhow::anyhow!("decode: {e}"))?;
match msg {
ClientMessage::Voucher(_) | ClientMessage::ChunkPreimage(_) => Ok(msg),
other => anyhow::bail!("scripted upstream: expected a payment proof, got {other:?}"),
}
}
async fn read_voucher(recv: &mut iroh::endpoint::RecvStream) -> Result<()> {
read_proof(recv).await.map(|_| ())
}
#[allow(clippy::too_many_arguments)]
async fn serve_wire_paced(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
wire: &[u8],
total_bytes: u64,
rate: u64,
gap: Duration,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req_msg = {
let frame = read_frame(&mut recv)
.await
.map_err(|e| anyhow::anyhow!("read request: {e}"))?;
decode_message::<ClientMessage>(&frame)
.map_err(|e| anyhow::anyhow!("decode request: {e}"))?
.0
};
let ClientMessage::StreamRequest(req) = req_msg else {
anyhow::bail!("paced upstream: expected a StreamRequest");
};
let body = StreamResponseBody {
hash: req.hash,
ok: true,
rate_per_mb: rate,
total_bytes,
pool_id: req.pool_id,
timestamp_us: req.timestamp_us,
};
let slash_sig = StreamSlashData::from_response_body(&body)
.sign(eth.as_ref(), slash)
.map_err(|e| anyhow::anyhow!("slash sign: {e}"))?
.as_bytes()
.to_vec();
let resp = StreamResponse { body, slash_sig };
write_frame(
&mut send,
&encode_stream_response(&resp, Some(&StreamResponseExt { error: None }))?,
)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
let frames = wire.len().div_ceil(WIRE_FRAME);
let mut wire_sent: u64 = 0;
let mut settled: u64 = 0;
for (ix, chunk) in wire.chunks(WIRE_FRAME).enumerate() {
if !gap.is_zero() {
tokio::time::sleep(gap).await;
}
write_frame(
&mut send,
&encode_message(&ClientMessage::ChunkData(ChunkData::new(chunk.to_vec())?))?,
)
.await
.map_err(|e| anyhow::anyhow!("write chunk: {e}"))?;
wire_sent = wire_sent.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX));
while let Some(served) = payable_interval(wire_sent, settled, ix + 1 == frames) {
read_voucher(&mut recv).await?;
settled = settled.saturating_add(served);
}
}
write_frame(&mut send, &encode_message(&ClientMessage::StreamEnd)?)
.await
.map_err(|e| anyhow::anyhow!("write end: {e}"))?;
let _ = send.finish();
conn.closed().await;
Ok(())
}
async fn spawn_paced_lying_node_a(
wire: Vec<u8>,
total_bytes: u64,
) -> Result<(
iroh::PublicKey,
std::net::SocketAddr,
Arc<PrivateKeySigner>,
iroh::Endpoint,
tokio::task::JoinHandle<()>,
)> {
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let slash = slash_domain();
let eth = Arc::clone(&a_eth);
let accept_ep = ep_a.clone();
let task_a = tokio::spawn(async move {
while let Some(incoming) = accept_ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(ð);
let dom = slash.clone();
let wire = wire.clone();
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, RATE, total_bytes).await;
});
} else {
tokio::spawn(async move {
let _ = serve_wire_paced(
conn,
ð,
&dom,
&wire,
total_bytes,
RATE,
Duration::ZERO,
)
.await;
});
}
}
});
Ok((a_id, addr_a, a_eth, ep_a, task_a))
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_lying_upstream_is_not_cached() -> Result<()> {
let honest = vec![0x77u8; 4096];
let hash = Hash::new(&honest);
let served_wrong = vec![0x88u8; 4096];
anyhow::ensure!(Hash::new(&served_wrong) != hash, "fixtures must differ");
let advertised_bytes = u64::try_from(honest.len()).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA6);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_lying_node_a(served_wrong, advertised_bytes).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x6F);
let (handler_b, b_target, ep_b, _recorded, cache_b, b_metrics, local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let a_score_before = local_rep.score(a_id);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await;
if let Ok(o) = &outcome {
anyhow::ensure!(
!o.hash_ok,
"a corrupt-upstream serve must never deliver a hash-verified blob"
);
}
anyhow::ensure!(
!cache_b.has(hash).await?,
"B must not promote a blob that failed upstream verification"
);
assert_counter(
&b_metrics,
"node_pull_through_upstream_verify_failed_total",
1,
)?;
let a_score_after = local_rep.score(a_id);
anyhow::ensure!(
a_score_after < a_score_before,
"a wire-complete corrupt upstream must be scored Corruption (score {a_score_before} -> {a_score_after})"
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_mid_stream_corruption_scores_upstream_not_local() -> Result<()> {
let spans = support::capture_spans();
let payload_len = usize::try_from(CHUNK_BYTES.saturating_mul(3) / 2).unwrap_or(usize::MAX);
let payload = vec![0xB7u8; payload_len];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload_len).unwrap_or(u64::MAX);
let ob = bao_tree::io::outboard::PreOrderMemOutboard::create(
&payload,
decdn_cache::range_pull::IROH_BLOCK_SIZE,
);
let aligned = decdn_cache::range_pull::align_range(0, 0, total_bytes)?;
let combined = decdn_cache::range_pull::encode_verified_range(
*hash.as_bytes(),
&aligned,
&payload,
bytes::Bytes::from(ob.data),
)?;
let mut wire = combined
.get(8..)
.ok_or_else(|| anyhow::anyhow!("combined encoding shorter than its header"))?
.to_vec();
let corrupt_at = usize::try_from(CHUNK_BYTES.saturating_add(400_000)).unwrap_or(usize::MAX);
let byte = wire
.get_mut(corrupt_at)
.ok_or_else(|| anyhow::anyhow!("corruption offset outside the wire"))?;
*byte ^= 0xFF;
let ab_pool_id = B256::repeat_byte(0xA7);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) = spawn_paced_lying_node_a(wire, total_bytes).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x7A);
let (handler_b, b_target, ep_b, _recorded, cache_b, b_metrics, local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let a_score_before = local_rep.score(a_id);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await;
if let Ok(o) = &outcome {
anyhow::ensure!(
!o.hash_ok,
"a mid-stream-corrupt serve must never deliver a verified blob"
);
}
anyhow::ensure!(
!cache_b.has(hash).await?,
"B must not promote a blob whose tee rejected a group mid-stream"
);
assert_counter(
&b_metrics,
"node_pull_through_upstream_verify_failed_total",
1,
)?;
let a_score_after = local_rep.score(a_id);
anyhow::ensure!(
a_score_after < a_score_before,
"mid-stream corruption must be scored Corruption (score {a_score_before} -> {a_score_after})"
);
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
await_run_outcome(&spans, hash, "reassigned", a_id, ab_pool_id).await?;
Ok(())
}
struct LeafStream {
conn: iroh::endpoint::Connection,
send: iroh::endpoint::SendStream,
recv: iroh::endpoint::RecvStream,
delivered: u64,
}
async fn leaf_reads_first_interval(
leaf_ep: &iroh::Endpoint,
target: EndpointAddr,
leaf_node_id: B256,
leaf_eth: &Arc<PrivateKeySigner>,
pool_id: B256,
hash: Hash,
) -> Result<LeafStream> {
let LeafStream {
conn,
send,
mut recv,
..
} = leaf_opens_paid_stream(leaf_ep, target, leaf_node_id, leaf_eth, pool_id, hash).await?;
let parks_at = CHUNK_BYTES + decdn_client::VERIFY_LAG_BYTES;
let mut delivered: u64 = 0;
loop {
match read_client(&mut recv).await? {
ClientMessage::ChunkData(chunk) => {
delivered = delivered.saturating_add(chunk.bytes().len() as u64);
if delivered >= parks_at {
break;
}
}
ClientMessage::StreamEnd => anyhow::bail!("stream ended before the first interval"),
other => anyhow::bail!("unexpected message mid-delivery: {other:?}"),
}
}
Ok(LeafStream {
conn,
send,
recv,
delivered,
})
}
async fn leaf_opens_paid_stream(
leaf_ep: &iroh::Endpoint,
target: EndpointAddr,
leaf_node_id: B256,
leaf_eth: &Arc<PrivateKeySigner>,
pool_id: B256,
hash: Hash,
) -> Result<LeafStream> {
let conn = leaf_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("leaf connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("leaf open_bi: {e}"))?;
let binding_hash = binding_signing_hash(leaf_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = {
use alloy::signers::SignerSync;
leaf_eth.sign_hash_sync(&binding_hash)?.as_bytes().to_vec()
};
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: leaf_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset: 0,
byte_len: 0,
timestamp_us: 0x9002,
};
write_frame(
&mut send,
&encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?,
)
.await
.map_err(|e| anyhow::anyhow!("write req: {e}"))?;
let (resp, resp_ext) = read_client_response(&mut recv).await?;
anyhow::ensure!(resp.body.ok, "delivery refused: {:?}", resp_ext.error);
Ok(LeafStream {
conn,
send,
recv,
delivered: 0,
})
}
async fn leaf_sends_voucher(
send: &mut iroh::endpoint::SendStream,
leaf_eth: &Arc<PrivateKeySigner>,
provider: Address,
pool_id: B256,
bytes_delivered: u64,
amount: U256,
) -> Result<()> {
let signed = Voucher {
pool_id,
signer: leaf_eth.address(),
provider,
amount,
bytes_delivered: U256::from(bytes_delivered),
chain_root: B256::ZERO,
chunk_price: U256::ZERO,
}
.sign(leaf_eth.as_ref(), &voucher_dom())
.map_err(|e| anyhow::anyhow!("sign voucher: {e}"))?;
write_client(
send,
&ClientMessage::Voucher(signed_to_wire_voucher(&signed)?),
)
.await
}
async fn leaf_underpays_first_voucher(
leaf_ep: &iroh::Endpoint,
target: EndpointAddr,
leaf_node_id: B256,
leaf_eth: &Arc<PrivateKeySigner>,
provider: Address,
pool_id: B256,
hash: Hash,
) -> Result<()> {
let mut leaf =
leaf_reads_first_interval(leaf_ep, target, leaf_node_id, leaf_eth, pool_id, hash).await?;
leaf_sends_voucher(
&mut leaf.send,
leaf_eth,
provider,
pool_id,
leaf.delivered,
U256::from(1u64),
)
.await?;
match read_client(&mut leaf.recv).await? {
ClientMessage::StreamError(StreamError::VoucherRejected {
reason: VoucherRejectReason::Underpaid,
..
}) => Ok(()),
other => anyhow::bail!("expected an Underpaid rejection, got {other:?}"),
}
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_underpaid_voucher_abandons_bounded() -> Result<()> {
let payload_len = usize::try_from(CHUNK_BYTES.saturating_mul(3) / 2).unwrap_or(usize::MAX);
let payload = vec![0x7Cu8; payload_len];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA7);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x7F);
let (handler_b, b_target, ep_b, recorded, cache_b, b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
leaf_underpays_first_voucher(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
)
.await?;
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
let settled = loop {
if let Some(&(_, bytes, _)) = progress_log(&recorded)?.last() {
break bytes;
}
anyhow::ensure!(
tokio::time::Instant::now() < deadline,
"B never recorded an upstream watermark: the pull leg did not settle"
);
tokio::time::sleep(Duration::from_millis(20)).await;
};
anyhow::ensure!(
!cache_b.has(hash).await?,
"an underpaid serve must not promote the partial blob"
);
let upstream_bytes: u64 = u64::try_from(settled).unwrap_or(u64::MAX);
let one_window = decdn_client::PULL_WINDOW_FLOOR;
let window_wire = support::bao_wire_len(
u64::try_from(payload_len).unwrap_or(u64::MAX),
0,
one_window,
);
anyhow::ensure!(
upstream_bytes <= window_wire,
"B's upstream spend ({upstream_bytes} wire) must stay bounded to one credit \
window ({one_window} content = {window_wire} wire)"
);
assert_counter(&b_metrics, "node_pull_through_client_abandoned_total", 1)?;
assert_counter(&b_metrics, "serve_stream_proof_budget_exhausted_total", 0)?;
support::assert_inbound_failures_attributed(&b_metrics, 1).await?;
assert_counter(&b_metrics, "serve_stream_voucher_rejected_total", 1)?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_sliver_voucher_leaves_the_rest_owed() -> Result<()> {
const SLIVER_BYTES: u64 = 64 * 1024;
let payload_len = usize::try_from(CHUNK_BYTES.saturating_mul(3)).unwrap_or(usize::MAX);
let payload = vec![0x2Du8; payload_len];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA8);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x2E);
let (handler_b, b_target, ep_b, _recorded, _cache_b, _b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let mut leaf = leaf_reads_first_interval(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
leaf_pool_id,
hash,
)
.await?;
let mut amount = min_payment(SLIVER_BYTES, RATE);
leaf_sends_voucher(
&mut leaf.send,
&leaf_eth,
b_operator,
leaf_pool_id,
SLIVER_BYTES,
amount,
)
.await?;
let mut reader = BufferedClientReader::default();
match tokio::time::timeout(Duration::from_secs(2), reader.read(&mut leaf.recv)).await {
Err(_elapsed) => {}
Ok(Ok(ClientMessage::ChunkData(chunk))) => anyhow::bail!(
"B sent {} more wire bytes on a sliver payment instead of waiting for the rest",
chunk.bytes().len()
),
Ok(other) => anyhow::bail!("expected B to wait for the rest of the payment, got {other:?}"),
}
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
let mut paid = SLIVER_BYTES;
loop {
anyhow::ensure!(
tokio::time::Instant::now() < deadline,
"B never finished the stream ({} bytes read, {paid} paid)",
leaf.delivered
);
match tokio::time::timeout(Duration::from_millis(500), reader.read(&mut leaf.recv)).await {
Ok(Ok(ClientMessage::ChunkData(chunk))) => {
leaf.delivered = leaf.delivered.saturating_add(chunk.bytes().len() as u64);
}
Ok(Ok(ClientMessage::StreamEnd)) => break,
Err(_quiet) if leaf.delivered == paid => {}
Err(_quiet) => {
let whole = leaf.delivered / CHUNK_BYTES * CHUNK_BYTES;
let upto = if whole > paid { whole } else { leaf.delivered };
amount += min_payment(upto - paid, RATE);
paid = upto;
leaf_sends_voucher(
&mut leaf.send,
&leaf_eth,
b_operator,
leaf_pool_id,
paid,
amount,
)
.await?;
}
Ok(other) => anyhow::bail!("expected the stream to run to StreamEnd, got {other:?}"),
}
}
anyhow::ensure!(
paid == leaf.delivered,
"B ended the stream with {} unpaid bytes",
leaf.delivered - paid
);
leaf.conn.close(0u32.into(), b"done");
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_sliver_vouchers_exhaust_the_proof_budget() -> Result<()> {
const SLIVER_BYTES: u64 = 64 * 1024;
const PROOF_BUDGET: u64 = 8;
anyhow::ensure!(
PROOF_BUDGET * SLIVER_BYTES < CHUNK_BYTES,
"the slivers must not add up to a whole interval"
);
let payload_len = usize::try_from(CHUNK_BYTES.saturating_mul(3)).unwrap_or(usize::MAX);
let payload = vec![0x3Eu8; payload_len];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xA9);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x3F);
let (handler_b, b_target, ep_b, _recorded, _cache_b, b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let mut leaf = leaf_reads_first_interval(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
leaf_pool_id,
hash,
)
.await?;
let mut amount = U256::ZERO;
for n in 1..=PROOF_BUDGET {
amount += min_payment(SLIVER_BYTES, RATE);
leaf_sends_voucher(
&mut leaf.send,
&leaf_eth,
b_operator,
leaf_pool_id,
n * SLIVER_BYTES,
amount,
)
.await?;
}
match tokio::time::timeout(Duration::from_secs(10), read_client(&mut leaf.recv)).await {
Err(_elapsed) => anyhow::bail!("B kept the stream open past the proof budget"),
Ok(Ok(ClientMessage::ChunkData(chunk))) => anyhow::bail!(
"B delivered {} more wire bytes on a chunk that is still owed",
chunk.bytes().len()
),
Ok(Ok(other)) => anyhow::bail!("expected the stream to fail, got {other:?}"),
Ok(Err(_)) => {}
}
assert_counter(&b_metrics, "node_pull_through_client_abandoned_total", 1)?;
assert_counter(&b_metrics, "serve_stream_proof_budget_exhausted_total", 1)?;
support::assert_inbound_failures_attributed(&b_metrics, 1).await?;
leaf.conn.close(0u32.into(), b"done");
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_store_record_failure_is_a_node_fault_not_an_abandon() -> Result<()> {
let payload_len = usize::try_from(CHUNK_BYTES.saturating_mul(3)).unwrap_or(usize::MAX);
let payload = vec![0x4Cu8; payload_len];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xAA);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x4D);
let (handler_b, b_target, ep_b, _recorded, _cache_b, b_metrics, _local_rep, b_operator) =
build_node_b_with_store(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
&[(
leaf_pool_id,
leaf_eth.address(),
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
)],
0,
64,
None,
DEFAULT_TEST_PULL_DEADLINES,
|store| {
Arc::new(support::FailingRecordStore { inner: store }) as Arc<dyn PoolStateStore>
},
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let mut leaf = leaf_reads_first_interval(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
leaf_pool_id,
hash,
)
.await?;
leaf_sends_voucher(
&mut leaf.send,
&leaf_eth,
b_operator,
leaf_pool_id,
CHUNK_BYTES,
min_payment(CHUNK_BYTES, RATE),
)
.await?;
match tokio::time::timeout(Duration::from_secs(10), read_client(&mut leaf.recv)).await {
Err(_elapsed) => anyhow::bail!("B kept the stream open past a lane-store fault"),
Ok(Ok(msg)) => anyhow::bail!("expected the stream to fail, got {msg:?}"),
Ok(Err(_)) => {}
}
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
while assert_counter(&b_metrics, "serve_stream_node_fault_total", 1).is_err()
&& tokio::time::Instant::now() < deadline
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert_counter(&b_metrics, "serve_stream_node_fault_total", 1)?;
assert_counter(&b_metrics, "node_pull_through_client_abandoned_total", 0)?;
assert_counter(&b_metrics, "serve_stream_proof_budget_exhausted_total", 0)?;
support::assert_inbound_failures_attributed(&b_metrics, 1).await?;
leaf.conn.close(0u32.into(), b"done");
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_leaf_drop_before_paying_is_an_abandon_and_a_decline() -> Result<()> {
let payload_len = usize::try_from(CHUNK_BYTES.saturating_mul(3)).unwrap_or(usize::MAX);
let payload = vec![0x5Eu8; payload_len];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xAB);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x5F);
let (handler_b, b_target, ep_b, _recorded, _cache_b, b_metrics, _local_rep, _b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let leaf = leaf_reads_first_interval(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
leaf_pool_id,
hash,
)
.await?;
leaf.conn.close(0u32.into(), b"gone");
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
while assert_counter(&b_metrics, "streams_failed_total{direction=\"inbound\"}", 1).is_err()
&& tokio::time::Instant::now() < deadline
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert_counter(&b_metrics, "streams_failed_total{direction=\"inbound\"}", 1)?;
assert_counter(&b_metrics, "node_pull_through_client_abandoned_total", 1)?;
assert_counter(&b_metrics, "serve_stream_proof_budget_exhausted_total", 0)?;
assert_counter(&b_metrics, "serve_stream_node_fault_total", 0)?;
assert_counter(&b_metrics, "serve_stream_client_declined_total", 1)?;
support::assert_inbound_failures_attributed(&b_metrics, 1).await?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_leaf_gone_while_the_pull_owes_the_first_byte_ends_the_serve()
-> Result<()> {
let payload_len = usize::try_from(CHUNK_BYTES.saturating_mul(3)).unwrap_or(usize::MAX);
let payload = vec![0x3Cu8; payload_len];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload_len).unwrap_or(u64::MAX);
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, a_addr) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_mid_stream_silent_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
Arc::new(honest_bao_wire(&payload)?),
total_bytes,
RATE,
0,
);
let ab_pool_id = B256::repeat_byte(0x3C);
let b_buyer = Arc::new(PrivateKeySigner::random());
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x3D);
let (handler_b, b_target, ep_b, _recorded, cache_b, b_metrics, _local_rep, _b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let leaf = leaf_opens_paid_stream(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
leaf_pool_id,
hash,
)
.await?;
anyhow::ensure!(
cache_b.in_flight_total(hash).is_some(),
"B's serve waits on a live fill"
);
leaf.conn.close(0u32.into(), b"gone");
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
while (assert_counter(&b_metrics, "streams_failed_total{direction=\"inbound\"}", 1).is_err()
|| cache_b.in_flight_total(hash).is_some())
&& tokio::time::Instant::now() < deadline
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert_counter(&b_metrics, "streams_failed_total{direction=\"inbound\"}", 1)?;
anyhow::ensure!(
cache_b.in_flight_total(hash).is_none(),
"the serve's lease release cancels the pull"
);
assert_counter(&b_metrics, "node_pull_through_client_abandoned_total", 1)?;
assert_counter(&b_metrics, "serve_stream_node_fault_total", 0)?;
assert_counter(&b_metrics, "serve_stream_client_declined_total", 1)?;
assert_counter(&b_metrics, "node_pull_leg_no_progress_total", 0)?;
support::assert_inbound_failures_attributed(&b_metrics, 1).await?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_leaf_drop_after_paying_is_a_serve_abandon() -> Result<()> {
let payload_len = usize::try_from(CHUNK_BYTES.saturating_mul(4)).unwrap_or(usize::MAX);
let payload = vec![0x6Au8; payload_len];
let hash = Hash::new(&payload);
let ab_pool_id = B256::repeat_byte(0xAB);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x6B);
let (handler_b, b_target, ep_b, _recorded, _cache_b, b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = leaf_paced_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
Some(2),
)
.await?;
anyhow::ensure!(
!outcome.completed && outcome.acks == 2,
"the leaf must drop after its second voucher"
);
support::assert_inbound_failures_attributed(&b_metrics, 1).await?;
assert_counter(&b_metrics, "serve_stream_client_abandoned_total", 1)?;
assert_counter(&b_metrics, "serve_stream_client_declined_total", 0)?;
assert_counter(&b_metrics, "serve_stream_node_fault_total", 0)?;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn window_pull_through_oversized_upstream_aborts_on_received_bytes() -> Result<()> {
let payload = vec![0xB1u8; 4 * 1024 * 1024];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA6);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a) =
spawn_node_a(&payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x6F);
let retry_eth = Arc::new(PrivateKeySigner::random());
let retry_pool_id = B256::repeat_byte(0x70);
let max_blob_size_bytes = 2 * CHUNK_BYTES;
let (handler_b, b_target, ep_b, recorded, cache_b, b_metrics, _local_rep, b_operator) =
build_node_b_with_leaves(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
&[
(
leaf_pool_id,
leaf_eth.address(),
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
),
(
retry_pool_id,
retry_eth.address(),
retry_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
),
],
max_blob_size_bytes,
64,
None,
DEFAULT_TEST_PULL_DEADLINES,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let refused = leaf_paced_pull(
&leaf_ep,
b_target.clone(),
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await
.is_err();
anyhow::ensure!(
refused,
"an upstream blob over the size ceiling must abort the fused serve mid-stream, not complete"
);
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
let settled = loop {
if let Some(&(_, bytes, _)) = progress_log(&recorded)?.last() {
break bytes;
}
anyhow::ensure!(
tokio::time::Instant::now() < deadline,
"B never recorded an upstream watermark: the pull leg did not settle"
);
tokio::time::sleep(Duration::from_millis(20)).await;
};
anyhow::ensure!(
!cache_b.has(hash).await?,
"an oversized-upstream abort must not promote the blob into B's cache"
);
let upstream_bytes: u64 = u64::try_from(settled).unwrap_or(u64::MAX);
let bound_content = max_blob_size_bytes;
let bound_wire = support::bao_wire_len(total_bytes, 0, bound_content);
anyhow::ensure!(
upstream_bytes > 0,
"B must pay the upstream for the bytes it actually received before aborting"
);
anyhow::ensure!(
upstream_bytes <= bound_wire,
"B's upstream spend ({upstream_bytes} wire) must stay bounded to ~one ceiling \
({bound_content} content = {bound_wire} wire), not the whole blob"
);
assert_counter(&b_metrics, "node_pull_too_large_total", 1)?;
let retry_sk = fresh_key();
let retry_node_id = B256::from(*retry_sk.public().as_bytes());
let (retry_ep, _) = local_endpoint(retry_sk, vec![]).await?;
let refused_again = leaf_paced_pull(
&retry_ep,
b_target,
retry_node_id,
&retry_eth,
b_operator,
retry_pool_id,
hash,
RATE,
None,
)
.await
.is_err();
anyhow::ensure!(
refused_again,
"a repeat request for the same hash must abort again, not wedge on a stranded tee claim"
);
assert_counter(&b_metrics, "node_pull_too_large_total", 2)?;
shutdown([task_a, task_b], [&retry_ep, &leaf_ep, &ep_b, &ep_a]).await?;
Ok(())
}
struct CeilingRun {
leaf: Result<Vec<u8>>,
b_metrics: Arc<Metrics>,
a_metrics: Arc<Metrics>,
upstream: Vec<ProgressEntry>,
}
async fn ranged_pull_under_ceiling(
payload: &[u8],
ceiling: u64,
byte_offset: u64,
byte_len: u64,
) -> Result<CeilingRun> {
let hash = Hash::new(payload);
let ab_pool_id = B256::repeat_byte(0xC7);
let b_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a, a_metrics) =
spawn_node_a_metered(payload, ab_pool_id, b_buyer.address()).await?;
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x7C);
let (handler_b, b_target, ep_b, recorded, _cache_b, b_metrics, _local_rep, b_operator) =
build_node_b(
a_id,
a_addr,
a_eth.address(),
hash,
ab_pool_id,
&b_buyer,
leaf_pool_id,
leaf_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
ceiling,
)
.await?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let leaf = leaf_ranged_paid_pull(
&leaf_ep,
b_target,
leaf_node_id,
&leaf_eth,
b_operator,
leaf_pool_id,
hash,
byte_offset,
byte_len,
RATE,
)
.await;
shutdown([task_a, task_b], [&leaf_ep, &ep_b, &ep_a]).await?;
let upstream = progress_log(&recorded)?;
Ok(CeilingRun {
leaf,
b_metrics,
a_metrics,
upstream,
})
}
async fn assert_refused_too_large(run: &CeilingRun) -> Result<()> {
let err = match &run.leaf {
Ok(got) => anyhow::bail!(
"an over-ceiling range must be refused, got {} bytes",
got.len()
),
Err(e) => format!("{e:#}"),
};
anyhow::ensure!(
err.contains("delivery refused: Some(Declined)"),
"the leaf must read a signed Declined refusal, got: {err}"
);
assert_counter(
&run.b_metrics,
"serve_stream_rejected_blob_too_large_total",
1,
)?;
assert_counter(&run.b_metrics, "node_pull_too_large_total", 0)?;
support::assert_inbound_failures_attributed(&run.b_metrics, 1).await?;
anyhow::ensure!(
run.upstream.iter().all(|(_, amount, _)| amount.is_zero()),
"B must pay the upstream nothing for a range it refuses: {:?}",
run.upstream
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_offset_at_the_ceiling_is_refused_before_commit() -> Result<()> {
let payload = vec![0xC1u8; 256 * 1024];
let ceiling = 64 * 1024;
let run = ranged_pull_under_ceiling(&payload, ceiling, ceiling, 16 * 1024).await?;
assert_refused_too_large(&run).await?;
let a_served = counter_value(&run.a_metrics, "serve_cache_hit_total")?;
anyhow::ensure!(
a_served == 0,
"B must refuse before it opens any upstream stream: A served {a_served}"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_end_past_the_ceiling_commits_and_aborts_on_received_bytes()
-> Result<()> {
let payload = vec![0xC2u8; 256 * 1024];
let ceiling = 64 * 1024;
let run = ranged_pull_under_ceiling(&payload, ceiling, 32 * 1024, 64 * 1024).await?;
let err = match &run.leaf {
Ok(got) => anyhow::bail!(
"a range past the ceiling cannot complete, got {} bytes",
got.len()
),
Err(e) => format!("{e:#}"),
};
anyhow::ensure!(
!err.contains("delivery refused"),
"B must commit to a range that starts below its ceiling, got: {err}"
);
assert_counter(
&run.b_metrics,
"serve_stream_rejected_blob_too_large_total",
0,
)?;
let aborts = counter_value(&run.b_metrics, "node_pull_too_large_total")?;
anyhow::ensure!(
aborts >= 1,
"the received-byte ceiling must abort the pull, counted {aborts}"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn window_pull_through_over_long_range_on_a_blob_within_the_ceiling_is_served() -> Result<()>
{
let payload: Vec<u8> = (0..128 * 1024usize)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect();
let total = u64::try_from(payload.len())?;
let offset = 16 * 1024u64;
let run = ranged_pull_under_ceiling(&payload, total, offset, total * 8).await?;
let got = run
.leaf
.map_err(|e| anyhow::anyhow!("a blob within the ceiling must be served: {e:#}"))?;
let want = payload
.get(usize::try_from(offset)?..)
.ok_or_else(|| anyhow::anyhow!("offset out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"the over-long range must clamp to the blob's end, byte-exact"
);
assert_counter(
&run.b_metrics,
"serve_stream_rejected_blob_too_large_total",
0,
)?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)]
async fn two_concurrent_pulls_to_one_provider_share_the_lane_ledger() -> Result<()> {
let payload = vec![0xC1u8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let payload2 = vec![0xC2u8; PAYLOAD_LEN];
let hash2 = Hash::new(&payload2);
anyhow::ensure!(hash != hash2, "the two fixtures must be distinct blobs");
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, _tmp_a) = cache_with_blobs(&[payload.as_slice(), payload2.as_slice()]).await?;
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xC7);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_a = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics_a);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics_a,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for h in [hash, hash2] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*h.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let (providers, addr_map) = one_provider(a_dht, a_eth.address());
let recorded: Arc<Mutex<Vec<ProgressEntry>>> = Arc::new(Mutex::new(Vec::new()));
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::clone(&recorded),
}) as Arc<dyn PoolOpener>;
let (engine, _engine_tmp) = throwaway_engine().await?;
let origin = NodeOrigin::new();
origin.provision(NodeOriginDeps {
endpoint: ep_b.clone(),
dial_runtime: tokio::runtime::Handle::current(),
routing_table: Arc::new(Mutex::new(RoutingTable::new(DhtNodeId::from_bytes(
*b_id.as_bytes(),
)))),
staker_set: Arc::new(ConfigStakerSet::empty()) as Arc<dyn StakerSet>,
origin_directory: Arc::new(StaticOriginDirectory::new(HashMap::from([
(U256::ZERO, providers.clone()),
(U256::ZERO, providers),
]))) as Arc<dyn OriginDirectory>,
addr_resolver: Arc::new(StaticNodeAddressDirectory::new(addr_map))
as Arc<dyn NodeAddressResolver>,
buyer,
self_id: DhtNodeId::from_bytes(*b_id.as_bytes()),
slash_domain: slash_domain(),
bind_domain: binding_dom(),
local_rep: Arc::clone(&local_rep),
negative_cache: NegativeProbeCache::new(),
probe_cache: PositiveProbeCache::new(),
metrics: Arc::clone(&b_metrics),
registry_regions: Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())),
config: NodeOriginConfig {
probe_fanout: 5,
pull_timeout: DEFAULT_TEST_PULL_DEADLINES.0,
stall_window: DEFAULT_TEST_PULL_DEADLINES.1,
min_throughput_bps: 0,
max_blob_size_bytes: 0,
max_rate_per_mb: 0,
event_poll_interval: Duration::from_millis(50),
lookup: decdn_node::dht::LookupConfig::default(),
own_region: None,
serve_economics: std::sync::Arc::new(decdn_node::serve_economics::OffPolicy),
operator_shares: decdn_node::fee_shares::OperatorShares::new(6000),
frequency_estimator: None,
sell_rate_base: 0,
warming: std::sync::Arc::new(decdn_node::warming_allowance::WarmingAllowance::new(
1_000_000_000,
0,
)),
},
ledgers: Arc::new(decdn_node::buyer_ledgers::BuyerLedgers::default()),
wedged_providers: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
engine: engine.clone(),
});
let join_budget = Duration::from_mins(1);
let (first, second) = tokio::time::timeout(join_budget, async {
tokio::join!(
Origin::fetch(&origin, hash, u64::MAX),
Origin::fetch(&origin, hash2, u64::MAX),
)
})
.await
.map_err(|_| {
anyhow::anyhow!(
"concurrent ledger pulls hung past {join_budget:?} — likely \
POOL_OPEN_CALLER_BUDGET={POOL_OPEN_CALLER_BUDGET:?} expiry under llvm-cov \
contention, not AmountRegression; pending={} timeout={} stalled={} recorded={:?}",
counter_value(&b_metrics, "node_pull_pool_open_pending_total").unwrap_or(0),
counter_value(&b_metrics, "node_pull_timeout_total").unwrap_or(0),
counter_value(&b_metrics, "node_pull_stalled_total").unwrap_or(0),
recorded.lock().map_or_else(|_| Vec::new(), |v| v.clone())
)
})?;
let first = first.map_err(|e| anyhow::anyhow!("first concurrent fetch failed: {e}"))?;
anyhow::ensure!(
matches!(&first, OriginFetch::AlreadyAdmitted),
"first concurrent pull returned {first:?} (expected AlreadyAdmitted); AmountRegression \
would wedge the lane and cap cumulative at one pull's wire bytes, while \
POOL_OPEN_CALLER_BUDGET expiry increments node_pull_pool_open_pending_total and \
surfaces NotFound — check pending={} timeout={} stalled={} recorded={:?}",
counter_value(&b_metrics, "node_pull_pool_open_pending_total").unwrap_or(0),
counter_value(&b_metrics, "node_pull_timeout_total").unwrap_or(0),
counter_value(&b_metrics, "node_pull_stalled_total").unwrap_or(0),
recorded.lock().map_or_else(|_| Vec::new(), |v| v.clone())
);
let got1 = engine.get(hash).await?;
let second = second.map_err(|e| anyhow::anyhow!("second concurrent fetch failed: {e}"))?;
anyhow::ensure!(
matches!(&second, OriginFetch::AlreadyAdmitted),
"second concurrent pull returned {second:?} (expected AlreadyAdmitted); AmountRegression \
would wedge the lane and cap cumulative at one pull's wire bytes, while \
POOL_OPEN_CALLER_BUDGET expiry increments node_pull_pool_open_pending_total and \
surfaces NotFound — check pending={} timeout={} stalled={} recorded={:?}",
counter_value(&b_metrics, "node_pull_pool_open_pending_total").unwrap_or(0),
counter_value(&b_metrics, "node_pull_timeout_total").unwrap_or(0),
counter_value(&b_metrics, "node_pull_stalled_total").unwrap_or(0),
recorded.lock().map_or_else(|_| Vec::new(), |v| v.clone())
);
let got2 = engine.get(hash2).await?;
anyhow::ensure!(got1.as_ref() == payload.as_slice(), "blob 1 bytes mismatch");
anyhow::ensure!(
got2.as_ref() == payload2.as_slice(),
"blob 2 bytes mismatch"
);
let single_wire =
decdn_cache::range_pull::bao_encoded_size(total_bytes, &bao_tree::ChunkRanges::all());
let combined_wire = U256::from(single_wire).saturating_mul(U256::from(2u64));
let entries = recorded.lock().expect("recorded lock").clone();
anyhow::ensure!(
entries.len() == 2,
"expected one recorded settlement per pull, got {entries:?}"
);
let top = entries
.iter()
.map(|(_, bytes, ..)| *bytes)
.max()
.unwrap_or(U256::ZERO);
anyhow::ensure!(
top == combined_wire,
"the shared ledger's cumulative must carry both pulls' wire bytes \
({combined_wire}); separate ledgers would each cap at one pull's {single_wire}: \
{entries:?}"
);
shutdown([task_a], [&ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn a_second_fetch_inside_the_ttl_skips_the_probe_entirely() -> Result<()> {
let payload = vec![0x5Cu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0x5C);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_a = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics_a);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics_a,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let probes = Arc::new(AtomicUsize::new(0));
let task_a = spawn_a_probe_counting_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
Arc::clone(&probes),
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
probes.store(0, Ordering::SeqCst);
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, engine, _recorded, _engine_tmp) = provisioned_origin(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
pool_id,
&b_buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
)
.await;
let first = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch: {e}"))?;
anyhow::ensure!(
matches!(first, OriginFetch::AlreadyAdmitted)
&& engine.get(hash).await?.as_ref() == payload.as_slice(),
"first fetch must deliver the blob"
);
anyhow::ensure!(probes.load(Ordering::SeqCst) == 1, "first fetch must probe");
assert_counter(&b_metrics, "probe_cache_misses_total", 1)?;
let second = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
anyhow::ensure!(
matches!(second, OriginFetch::AlreadyAdmitted)
&& engine.get(hash).await?.as_ref() == payload.as_slice(),
"second fetch must deliver the blob"
);
anyhow::ensure!(
probes.load(Ordering::SeqCst) == 1,
"the second fetch re-probed — the probe cache saved nothing, which is the entire \
point of ADR 001 §Probe cache"
);
assert_counter(&b_metrics, "probe_cache_hits_total", 1)?;
assert_counter(&b_metrics, "probe_cache_misses_total", 1)?;
assert_counter(&b_metrics, "probe_collection_latency_seconds_count", 1)?;
assert_counter(&b_metrics, "node_pull_attempts_total", 2)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn a_fetch_past_the_ttl_probes_again() -> Result<()> {
let payload = vec![0x7Eu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0x7E);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_a = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics_a);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics_a,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let probes = Arc::new(AtomicUsize::new(0));
let task_a = spawn_a_probe_counting_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
Arc::clone(&probes),
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
probes.store(0, Ordering::SeqCst);
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let b_buyer2 = Arc::clone(&b_buyer);
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: b_buyer2,
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_with_probe_caches(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
NegativeProbeCache::new(),
PositiveProbeCache::with_capacity_and_ttl(16, Duration::from_millis(300)),
U256::ZERO,
)
.await;
let first = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch: {e}"))?;
anyhow::ensure!(
matches!(first, OriginFetch::AlreadyAdmitted)
&& engine.get(hash).await?.as_ref() == payload.as_slice(),
"first fetch must deliver the blob"
);
anyhow::ensure!(probes.load(Ordering::SeqCst) == 1, "first fetch must probe");
tokio::time::sleep(Duration::from_millis(900)).await;
let second = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
anyhow::ensure!(
matches!(second, OriginFetch::AlreadyAdmitted)
&& engine.get(hash).await?.as_ref() == payload.as_slice(),
"second fetch must deliver the blob"
);
anyhow::ensure!(
probes.load(Ordering::SeqCst) == 2,
"a fetch past the TTL must probe again — the entry is gone, got {} probes",
probes.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "probe_cache_misses_total", 2)?;
assert_counter(&b_metrics, "probe_cache_hits_total", 0)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn cached_candidates_and_the_cold_path_share_one_attempt_budget() -> Result<()> {
let payload = vec![0x3Bu8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let probes = Arc::new(AtomicUsize::new(0));
let streams = Arc::new(AtomicUsize::new(0));
let n0_sk = fresh_key();
let n0_id = n0_sk.public();
let n0_eth = Arc::new(PrivateKeySigner::random());
let (ep_n0, addr_n0) =
local_endpoint(n0_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_n0 = spawn_a_refusing_server_with_counters(
ep_n0.clone(),
Arc::clone(&n0_eth),
slash_domain(),
total_bytes,
RATE,
StreamError::Declined,
Arc::clone(&probes),
Arc::clone(&streams),
);
let n1_sk = fresh_key();
let n1_id = n1_sk.public();
let n1_eth = Arc::new(PrivateKeySigner::random());
let (ep_n1, addr_n1) =
local_endpoint(n1_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_n1 = spawn_a_refusing_server_with_counters(
ep_n1.clone(),
Arc::clone(&n1_eth),
slash_domain(),
total_bytes,
RATE,
StreamError::Declined,
Arc::clone(&probes),
Arc::clone(&streams),
);
let n2_sk = fresh_key();
let n2_id = n2_sk.public();
let n2_eth = Arc::new(PrivateKeySigner::random());
let (ep_n2, addr_n2) =
local_endpoint(n2_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_n2 = spawn_a_refusing_server_with_counters(
ep_n2.clone(),
Arc::clone(&n2_eth),
slash_domain(),
total_bytes,
RATE,
StreamError::Declined,
Arc::clone(&probes),
Arc::clone(&streams),
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for (id, addr) in [(n0_id, addr_n0), (n1_id, addr_n1), (n2_id, addr_n2)] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(id).with_ip_addr(addr),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
probes.store(0, Ordering::SeqCst);
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let n0_dht = DhtNodeId::from_bytes(*n0_id.as_bytes());
let n1_dht = DhtNodeId::from_bytes(*n1_id.as_bytes());
let n2_dht = DhtNodeId::from_bytes(*n2_id.as_bytes());
let mut addr_map = HashMap::new();
addr_map.insert(n0_dht, n0_eth.address());
addr_map.insert(n1_dht, n1_eth.address());
addr_map.insert(n2_dht, n2_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0x3B),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_negative_cache(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
vec![n0_dht, n1_dht, n2_dht],
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
NegativeProbeCache::with_capacity_and_ttl(64, Duration::ZERO),
)
.await;
let first = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch: {e}"))?;
anyhow::ensure!(
matches!(first, OriginFetch::NotFound),
"all three providers refuse; the first fetch must miss"
);
anyhow::ensure!(
probes.load(Ordering::SeqCst) == 3,
"expected all three providers probed exactly once, got {}",
probes.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "node_pull_refused_total", 3)?;
let second = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
anyhow::ensure!(
matches!(second, OriginFetch::NotFound),
"the cached candidates all refuse again; the second fetch must miss too"
);
anyhow::ensure!(
probes.load(Ordering::SeqCst) == 3,
"the second fetch re-probed — the cache hit must supply the whole budget without a \
fresh lookup, got {} probes",
probes.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "node_pull_refused_total", 6)?;
assert_counter(&b_metrics, "probe_cache_hits_total", 1)?;
assert_counter(&b_metrics, "probe_cache_misses_total", 1)?;
let third = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("third fetch: {e}"))?;
anyhow::ensure!(
matches!(third, OriginFetch::NotFound),
"all three providers still refuse; the third fetch must miss too"
);
anyhow::ensure!(
probes.load(Ordering::SeqCst) == 6,
"the third fetch must re-probe all three providers — the exhausted entry was \
disproved by real pulls and must not survive to hit again, got {} probes",
probes.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "probe_cache_hits_total", 1)?;
assert_counter(&b_metrics, "probe_cache_misses_total", 2)?;
assert_counter(&b_metrics, "node_pull_refused_total", 9)?;
assert_counter(&b_metrics, "node_pull_attempts_total", 3)?;
shutdown([task_n0, task_n1, task_n2], [&ep_b, &ep_n0, &ep_n1, &ep_n2]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn a_partial_cached_budget_falls_through_to_the_cold_path_and_meters_once() -> Result<()> {
let payload = vec![0x1Bu8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let probes_n = Arc::new(AtomicUsize::new(0));
let streams_n = Arc::new(AtomicUsize::new(0));
let n_sk = fresh_key();
let n_id = n_sk.public();
let n_eth = Arc::new(PrivateKeySigner::random());
let (ep_n, addr_n) =
local_endpoint(n_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_n = spawn_a_refusing_server_with_counters(
ep_n.clone(),
Arc::clone(&n_eth),
slash_domain(),
total_bytes,
RANK_LAST_RATE,
StreamError::Declined,
Arc::clone(&probes_n),
Arc::clone(&streams_n),
);
let h_sk = fresh_key();
let h_id = h_sk.public();
let h_eth = Arc::new(PrivateKeySigner::random());
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(n_id).with_ip_addr(addr_n),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
probes_n.store(0, Ordering::SeqCst);
streams_n.store(0, Ordering::SeqCst);
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0x1B);
let n_dht = DhtNodeId::from_bytes(*n_id.as_bytes());
let h_dht = DhtNodeId::from_bytes(*h_id.as_bytes());
let mut addr_map = HashMap::new();
addr_map.insert(n_dht, n_eth.address());
addr_map.insert(h_dht, h_eth.address());
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_with_negative_cache(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
vec![n_dht, h_dht],
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
NegativeProbeCache::with_capacity_and_ttl(64, Duration::ZERO),
)
.await;
let first = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch: {e}"))?;
anyhow::ensure!(
matches!(first, OriginFetch::NotFound),
"H is unreachable and N refuses; the first fetch must miss"
);
anyhow::ensure!(
probes_n.load(Ordering::SeqCst) == 1,
"expected N probed exactly once, got {}",
probes_n.load(Ordering::SeqCst)
);
anyhow::ensure!(
streams_n.load(Ordering::SeqCst) == 1,
"expected N pulled-from exactly once, got {}",
streams_n.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "probe_cache_misses_total", 1)?;
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 1)?;
assert_counter(&b_metrics, "node_pull_attempts_total", 1)?;
let (cache_h, hash_h, _tmp_h) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_h == hash, "fixture hash mismatch");
let store_h = Arc::new(MemoryPoolStateStore::new());
store_h.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
h_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_h_side = Arc::new(Metrics::new());
let limiter_h = permissive_limiter(&metrics_h_side);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let h_rate = 1u64;
let handler_h = build_handler_full(
h_id,
&h_eth,
&metrics_h_side,
limiter_h,
cache_h,
store_h as Arc<dyn PoolStateStore>,
h_rate,
&domains,
16,
)?;
let (ep_h, addr_h) =
local_endpoint(h_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let probes_h = Arc::new(AtomicUsize::new(0));
let task_h = spawn_a_probe_counting_server(
ep_h.clone(),
handler_h,
Arc::clone(&h_eth),
slash_domain(),
total_bytes,
h_rate,
Arc::clone(&probes_h),
);
let _ = probe_once(
&ep_b,
EndpointAddr::new(h_id).with_ip_addr(addr_h),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
probes_h.store(0, Ordering::SeqCst);
let second = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
anyhow::ensure!(
matches!(second, OriginFetch::AlreadyAdmitted)
&& engine.get(hash).await?.as_ref() == payload.as_slice(),
"the second fetch must deliver the blob via the cold fallthrough"
);
anyhow::ensure!(
streams_n.load(Ordering::SeqCst) == 2,
"N must be pulled-from exactly twice total (once per fetch) — the cold path must not \
retry it once H (ranked first) delivers, got {}",
streams_n.load(Ordering::SeqCst)
);
anyhow::ensure!(
probes_n.load(Ordering::SeqCst) == 2,
"N must be re-probed by the fresh cold-path lookup (the cache hit itself never probes), \
got {}",
probes_n.load(Ordering::SeqCst)
);
anyhow::ensure!(
probes_h.load(Ordering::SeqCst) == 1,
"H must be probed exactly once, by the cold-path fallthrough, got {}",
probes_h.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "probe_cache_hits_total", 1)?;
assert_counter(&b_metrics, "probe_cache_misses_total", 1)?;
assert_counter(&b_metrics, "node_pull_refused_total", 2)?;
assert_counter(&b_metrics, "node_pull_unreachable_total", 1)?;
assert_counter(&b_metrics, "node_pull_attempts_total", 2)?;
shutdown([task_n, task_h], [&ep_b, &ep_n, &ep_h]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn a_probe_cache_hit_still_honours_the_negative_cache() -> Result<()> {
let payload = vec![0x9Du8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let n_sk = fresh_key();
let n_id = n_sk.public();
let n_eth = Arc::new(PrivateKeySigner::random());
let (ep_n, addr_n) =
local_endpoint(n_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let n_probes = Arc::new(AtomicUsize::new(0));
let n_streams = Arc::new(AtomicUsize::new(0));
let task_n = spawn_a_refusing_server_with_counters(
ep_n.clone(),
Arc::clone(&n_eth),
slash_domain(),
total_bytes,
1,
StreamError::NotFound,
Arc::clone(&n_probes),
Arc::clone(&n_streams),
);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0x9D);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_a = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics_a);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics_a,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RANK_LAST_RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let a_probes = Arc::new(AtomicUsize::new(0));
let task_a = spawn_a_probe_counting_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RANK_LAST_RATE,
Arc::clone(&a_probes),
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for (id, addr) in [(n_id, addr_n), (a_id, addr_a)] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(id).with_ip_addr(addr),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
n_probes.store(0, Ordering::SeqCst);
a_probes.store(0, Ordering::SeqCst);
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let n_dht = DhtNodeId::from_bytes(*n_id.as_bytes());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let mut addr_map = HashMap::new();
addr_map.insert(n_dht, n_eth.address());
addr_map.insert(a_dht, a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
vec![n_dht, a_dht],
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
)
.await;
let first = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch: {e}"))?;
anyhow::ensure!(
matches!(first, OriginFetch::AlreadyAdmitted)
&& engine.get(hash).await?.as_ref() == payload.as_slice(),
"first fetch must deliver the blob from A"
);
anyhow::ensure!(
n_probes.load(Ordering::SeqCst) == 1,
"N must be probed exactly once, got {}",
n_probes.load(Ordering::SeqCst)
);
anyhow::ensure!(
n_streams.load(Ordering::SeqCst) == 1,
"N must be pulled exactly once, got {}",
n_streams.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
let second = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
anyhow::ensure!(
matches!(second, OriginFetch::AlreadyAdmitted)
&& engine.get(hash).await?.as_ref() == payload.as_slice(),
"second fetch must still deliver the blob from A"
);
anyhow::ensure!(
n_probes.load(Ordering::SeqCst) == 1,
"the cache hit re-probed N — the negative cache must be honoured on a hit, got {}",
n_probes.load(Ordering::SeqCst)
);
anyhow::ensure!(
n_streams.load(Ordering::SeqCst) == 1,
"the cache hit re-streamed N — the negative cache must be honoured on a hit, got {}",
n_streams.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
assert_counter(&b_metrics, "probe_cache_hits_total", 1)?;
assert_counter(&b_metrics, "probe_cache_misses_total", 1)?;
shutdown([task_n, task_a], [&ep_b, &ep_n, &ep_a]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn an_entry_whose_every_provider_is_suppressed_is_a_miss_not_a_hit() -> Result<()> {
let payload = vec![0x2Fu8; 4096];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let probes_n = Arc::new(AtomicUsize::new(0));
let streams_n = Arc::new(AtomicUsize::new(0));
let n_sk = fresh_key();
let n_id = n_sk.public();
let n_eth = Arc::new(PrivateKeySigner::random());
let (ep_n, addr_n) =
local_endpoint(n_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_n = spawn_a_refusing_server_with_counters(
ep_n.clone(),
Arc::clone(&n_eth),
slash_domain(),
total_bytes,
RATE,
StreamError::Declined,
Arc::clone(&probes_n),
Arc::clone(&streams_n),
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(n_id).with_ip_addr(addr_n),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
probes_n.store(0, Ordering::SeqCst);
streams_n.store(0, Ordering::SeqCst);
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let b_buyer = Arc::new(PrivateKeySigner::random());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*n_id.as_bytes()), n_eth.address());
let buyer = Arc::new(StubOpener {
pool_id: B256::repeat_byte(0x2F),
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, _engine, _engine_tmp) = build_origin_with_timeout(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
Duration::from_secs(20),
Duration::from_secs(20),
0,
)
.await;
let first = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch: {e}"))?;
anyhow::ensure!(
matches!(first, OriginFetch::NotFound),
"N refuses; the first fetch must miss"
);
anyhow::ensure!(
probes_n.load(Ordering::SeqCst) == 1,
"expected N probed exactly once, got {}",
probes_n.load(Ordering::SeqCst)
);
anyhow::ensure!(
streams_n.load(Ordering::SeqCst) == 1,
"expected N pulled-from exactly once, got {}",
streams_n.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "probe_cache_misses_total", 1)?;
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
let second = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
anyhow::ensure!(
matches!(second, OriginFetch::NotFound),
"N is still suppressed; the second fetch must miss"
);
assert_counter(&b_metrics, "probe_cache_hits_total", 0)?;
assert_counter(&b_metrics, "probe_cache_misses_total", 2)?;
anyhow::ensure!(
probes_n.load(Ordering::SeqCst) == 1,
"the suppressed N must not be re-probed by either phase of fetch #2, got {}",
probes_n.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "probe_collection_latency_seconds_count", 1)?;
anyhow::ensure!(
streams_n.load(Ordering::SeqCst) == 1,
"the suppressed N must not be re-streamed by fetch #2, got {}",
streams_n.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "node_pull_refused_total", 1)?;
assert_counter(&b_metrics, "node_pull_attempts_total", 2)?;
assert_counter(&b_metrics, "node_pull_no_providers_total", 0)?;
shutdown([task_n], [&ep_b, &ep_n]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn a_probe_cache_hit_still_honours_the_wedged_provider_filter() -> Result<()> {
let payload1 = vec![0x33u8; 4096];
let payload2 = vec![0x44u8; 4096];
let hash1 = Hash::new(&payload1);
let hash2 = Hash::new(&payload2);
let probes_a = Arc::new(AtomicUsize::new(0));
let streams_a = Arc::new(AtomicUsize::new(0));
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_voucher_rejecting_server_with_counters(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
payload1.clone(),
u64::try_from(payload1.len()).unwrap_or(u64::MAX),
RANK_LAST_RATE,
VoucherRejectReason::AmountRegression,
Arc::clone(&probes_a),
Arc::clone(&streams_a),
);
let (cache_h, hash_h, _tmp_h) = cache_with_blob(&payload2).await?;
anyhow::ensure!(hash_h == hash2, "fixture hash mismatch");
let h_sk = fresh_key();
let h_id = h_sk.public();
let h_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0x3D);
let store_h = Arc::new(MemoryPoolStateStore::new());
store_h.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
h_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_h_side = Arc::new(Metrics::new());
let limiter_h = permissive_limiter(&metrics_h_side);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let h_rate = 1u64;
let handler_h = build_handler_full(
h_id,
&h_eth,
&metrics_h_side,
limiter_h,
cache_h,
store_h as Arc<dyn PoolStateStore>,
h_rate,
&domains,
16,
)?;
let (ep_h, addr_h) =
local_endpoint(h_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let probes_h = Arc::new(AtomicUsize::new(0));
let task_h = spawn_a_probe_counting_server(
ep_h.clone(),
handler_h,
Arc::clone(&h_eth),
slash_domain(),
u64::try_from(payload2.len()).unwrap_or(u64::MAX),
h_rate,
Arc::clone(&probes_h),
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
for (id, addr) in [(a_id, addr_a), (h_id, addr_h)] {
let _ = probe_once(
&ep_b,
EndpointAddr::new(id).with_ip_addr(addr),
*hash2.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
probes_a.store(0, Ordering::SeqCst);
streams_a.store(0, Ordering::SeqCst);
probes_h.store(0, Ordering::SeqCst);
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let h_dht = DhtNodeId::from_bytes(*h_id.as_bytes());
let mut addr_map = HashMap::new();
addr_map.insert(a_dht, a_eth.address());
addr_map.insert(h_dht, h_eth.address());
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_multi_hash(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
&[hash1, hash2],
buyer,
&local_rep,
&b_metrics,
&[a_dht, h_dht],
addr_map,
Duration::from_secs(3),
Duration::from_secs(3),
)
.await;
let first = Origin::fetch(&origin, hash2, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch: {e}"))?;
anyhow::ensure!(
matches!(first, OriginFetch::AlreadyAdmitted)
&& engine.get(hash2).await?.as_ref() == payload2.as_slice(),
"fetch #1 must deliver hash2 from H"
);
anyhow::ensure!(
streams_a.load(Ordering::SeqCst) == 0,
"A must not be pulled for hash2 while H (ranked first) delivers, got {}",
streams_a.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "node_pull_pool_wedged_total", 0)?;
assert_counter(&b_metrics, "probe_cache_misses_total", 1)?;
let second = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash1, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("fetch #2 never ended"))?
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
anyhow::ensure!(
matches!(second, OriginFetch::NotFound),
"fetch #2 must refuse: H lacks hash1 and A cannot be paid"
);
anyhow::ensure!(
streams_a.load(Ordering::SeqCst) == 1,
"A must be pulled exactly once (the handshake the drive adopts, the hash1 wedge), \
got {}",
streams_a.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "node_pull_pool_wedged_total", 1)?;
assert_counter(&b_metrics, "node_pull_voucher_rejected_total", 1)?;
let probes_a_after_wedge = probes_a.load(Ordering::SeqCst);
shutdown([task_h], [&ep_h]).await?;
let third = tokio::time::timeout(
Duration::from_secs(30),
Origin::fetch(&origin, hash2, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("fetch #3 never ended"))?
.map_err(|e| anyhow::anyhow!("third fetch: {e}"))?;
anyhow::ensure!(
matches!(third, OriginFetch::NotFound),
"fetch #3 must miss: H is offline and A is wedged"
);
anyhow::ensure!(
streams_a.load(Ordering::SeqCst) == 1,
"the cache hit re-streamed the WEDGED A — `cached_candidates` must filter a wedged \
provider even when its (peer, hash) pair was never negative-cached, got {}",
streams_a.load(Ordering::SeqCst)
);
anyhow::ensure!(
probes_a.load(Ordering::SeqCst) == probes_a_after_wedge,
"the wedged A must not be re-probed, got {} (was {probes_a_after_wedge})",
probes_a.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "node_pull_pool_wedged_total", 1)?;
assert_counter(&b_metrics, "probe_cache_hits_total", 1)?;
assert_counter(&b_metrics, "probe_cache_misses_total", 2)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
#[derive(Debug)]
struct MutableStakerSet {
active: Mutex<HashSet<DhtNodeId>>,
}
impl MutableStakerSet {
const fn new(active: HashSet<DhtNodeId>) -> Self {
Self {
active: Mutex::new(active),
}
}
fn remove(&self, node_id: &DhtNodeId) {
self.active
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(node_id);
}
}
impl StakerSet for MutableStakerSet {
fn is_active(&self, node_id: &DhtNodeId) -> bool {
self.active
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains(node_id)
}
fn active_nodes(&self) -> Vec<DhtNodeId> {
self.active
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.iter()
.copied()
.collect()
}
}
#[derive(Debug)]
struct MutableOriginDirectory {
origins: Mutex<HashMap<U256, Vec<DhtNodeId>>>,
}
impl MutableOriginDirectory {
const fn new(origins: HashMap<U256, Vec<DhtNodeId>>) -> Self {
Self {
origins: Mutex::new(origins),
}
}
fn remove(&self, namespace: U256) {
self.origins
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&namespace);
}
}
#[async_trait]
impl OriginDirectory for MutableOriginDirectory {
async fn lookup_origins(&self, namespace_id: U256) -> Vec<DhtNodeId> {
self.origins
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&namespace_id)
.cloned()
.unwrap_or_default()
}
}
#[allow(clippy::too_many_arguments)]
fn spawn_a_serving_server_with_counters(
ep: iroh::Endpoint,
client_handler: Arc<decdn_node::handlers::client::ClientHandler>,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
total_bytes: u64,
rate: u64,
probes: Arc<AtomicUsize>,
streams: Arc<AtomicUsize>,
) -> tokio::task::JoinHandle<()> {
use iroh::protocol::ProtocolHandler;
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
if conn.alpn() == ALPN_PROBE {
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let probes = Arc::clone(&probes);
tokio::spawn(async move {
probes.fetch_add(1, Ordering::SeqCst);
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
let handler = Arc::clone(&client_handler);
let streams = Arc::clone(&streams);
tokio::spawn(async move {
streams.fetch_add(1, Ordering::SeqCst);
let _ = decdn_node::handlers::client::ClientProtocol::new(handler)
.accept(conn)
.await;
});
}
}
})
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)] async fn a_probe_cache_hit_drops_a_provider_no_longer_admitted() -> Result<()> {
let payload = vec![0xA7u8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let pool_id = B256::repeat_byte(0xA7);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics_a = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics_a);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&metrics_a,
limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let a_probes = Arc::new(AtomicUsize::new(0));
let a_streams = Arc::new(AtomicUsize::new(0));
let task_a = spawn_a_serving_server_with_counters(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
RATE,
Arc::clone(&a_probes),
Arc::clone(&a_streams),
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, _addr_b) = local_endpoint(b_sk, vec![]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
a_probes.store(0, Ordering::SeqCst);
a_streams.store(0, Ordering::SeqCst);
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let a_dht = DhtNodeId::from_bytes(*a_id.as_bytes());
let b_dht = DhtNodeId::from_bytes(*b_id.as_bytes());
let mutable_staker = Arc::new(MutableStakerSet::new(HashSet::from([a_dht])));
let mutable_dir = Arc::new(MutableOriginDirectory::new(HashMap::from([(
U256::ZERO,
vec![a_dht],
)])));
let mut addr_map = HashMap::new();
addr_map.insert(a_dht, a_eth.address());
let buyer = Arc::new(StubOpener {
pool_id,
deposit: U256::from(DEPOSIT_MICRO_USDC),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (engine, _engine_tmp) = throwaway_engine().await?;
let origin = NodeOrigin::new();
origin.provision(NodeOriginDeps {
endpoint: ep_b.clone(),
dial_runtime: tokio::runtime::Handle::current(),
routing_table: Arc::new(Mutex::new(RoutingTable::new(b_dht))),
staker_set: Arc::clone(&mutable_staker) as Arc<dyn StakerSet>,
origin_directory: Arc::clone(&mutable_dir) as Arc<dyn OriginDirectory>,
addr_resolver: Arc::new(StaticNodeAddressDirectory::new(addr_map))
as Arc<dyn NodeAddressResolver>,
buyer,
self_id: b_dht,
slash_domain: slash_domain(),
bind_domain: binding_dom(),
local_rep: Arc::clone(&local_rep),
negative_cache: NegativeProbeCache::new(),
probe_cache: PositiveProbeCache::new(),
metrics: Arc::clone(&b_metrics),
registry_regions: Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())),
config: NodeOriginConfig {
probe_fanout: 5,
pull_timeout: DEFAULT_TEST_PULL_DEADLINES.0,
stall_window: DEFAULT_TEST_PULL_DEADLINES.1,
min_throughput_bps: 0,
max_blob_size_bytes: 0,
max_rate_per_mb: 0,
event_poll_interval: Duration::from_millis(50),
lookup: decdn_node::dht::LookupConfig::default(),
own_region: None,
serve_economics: std::sync::Arc::new(decdn_node::serve_economics::OffPolicy),
operator_shares: decdn_node::fee_shares::OperatorShares::new(6000),
frequency_estimator: None,
sell_rate_base: 0,
warming: std::sync::Arc::new(decdn_node::warming_allowance::WarmingAllowance::new(
1_000_000_000,
0,
)),
},
ledgers: Arc::new(decdn_node::buyer_ledgers::BuyerLedgers::default()),
wedged_providers: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
engine: engine.clone(),
});
let first = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("first fetch: {e}"))?;
anyhow::ensure!(
matches!(first, OriginFetch::AlreadyAdmitted)
&& engine.get(hash).await?.as_ref() == payload.as_slice(),
"first fetch must deliver the blob from A"
);
anyhow::ensure!(
a_probes.load(Ordering::SeqCst) == 1,
"A must be probed exactly once on the cold path, got {}",
a_probes.load(Ordering::SeqCst)
);
anyhow::ensure!(
a_streams.load(Ordering::SeqCst) == 1,
"A must be pulled exactly once (the handshake the drive adopts) on the cold path, \
got {}",
a_streams.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "probe_cache_misses_total", 1)?;
assert_counter(&b_metrics, "probe_cache_hits_total", 0)?;
mutable_staker.remove(&a_dht);
mutable_dir.remove(U256::ZERO);
let second = Origin::fetch(&origin, hash, u64::MAX)
.await
.map_err(|e| anyhow::anyhow!("second fetch: {e}"))?;
anyhow::ensure!(
matches!(second, OriginFetch::NotFound),
"second fetch must miss — the only cached provider was ejected inside the TTL"
);
anyhow::ensure!(
a_streams.load(Ordering::SeqCst) == 1,
"the hit re-served an EJECTED A — `is_active` must deny it on the hit path, got {} \
streams",
a_streams.load(Ordering::SeqCst)
);
anyhow::ensure!(
a_probes.load(Ordering::SeqCst) == 1,
"the ejected A must not be re-probed, got {}",
a_probes.load(Ordering::SeqCst)
);
assert_counter(&b_metrics, "probe_cache_hits_total", 0)?;
assert_counter(&b_metrics, "probe_cache_misses_total", 2)?;
assert_counter(&b_metrics, "node_pull_no_providers_total", 1)?;
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok(())
}
fn honest_bao_wire_range(payload: &[u8], byte_offset: u64, byte_len: u64) -> Result<Vec<u8>> {
support::honest_bao_wire_range(payload, byte_offset, byte_len)
}
fn honest_bao_wire_from(payload: &[u8], byte_offset: u64) -> Result<Vec<u8>> {
support::honest_bao_wire_range(payload, byte_offset, 0)
}
type SharedDeposit = Arc<Mutex<U256>>;
fn shared_deposit(micro_usdc: u64) -> SharedDeposit {
Arc::new(Mutex::new(U256::from(micro_usdc)))
}
fn read_deposit(cell: &SharedDeposit) -> Result<U256> {
Ok(*cell
.lock()
.map_err(|_| anyhow::anyhow!("deposit lock poisoned"))?)
}
#[derive(Debug)]
struct FundingOpener {
pool_id: B256,
deposit: SharedDeposit,
ceiling: SharedDeposit,
ceiling_follows: bool,
signer: Arc<PrivateKeySigner>,
voucher_domain: Eip712Domain,
recorded: Arc<Mutex<Vec<ProgressEntry>>>,
topups: Arc<Mutex<Vec<(Address, U256)>>>,
working: U256,
funds: bool,
landing_cap: Option<U256>,
}
#[async_trait]
impl PoolOpener for FundingOpener {
async fn open_or_reuse_pool(
&self,
provider_addr: Address,
_budget: Duration,
) -> Result<PoolContext> {
let recorded = self
.recorded
.lock()
.map_err(|_| anyhow::anyhow!("recorded lock poisoned"))?;
let (prior_bytes_delivered, prior_amount) = recorded
.iter()
.rev()
.find(|(provider, ..)| *provider == provider_addr)
.map_or((U256::ZERO, U256::ZERO), |(_, b, a)| (*b, *a));
drop(recorded);
Ok(PoolContext {
pool_id: self.pool_id,
provider: provider_addr,
deposit: read_deposit(&self.deposit)?,
client_signer: Arc::clone(&self.signer),
voucher_domain: self.voucher_domain.clone(),
prior_bytes_delivered,
prior_amount,
client_binding: None,
capability: None,
})
}
fn record_progress(
&self,
provider_addr: Address,
pool_id: B256,
write: decdn_client::buyer_pool::ProgressWrite,
) -> Result<()> {
let totals = write.totals();
let (bytes_delivered, amount) = (totals.last_bytes, totals.last_amount);
anyhow::ensure!(
pool_id == self.pool_id,
"record_progress pool_id {pool_id} != opened pool {}",
self.pool_id
);
self.recorded
.lock()
.map_err(|_| anyhow::anyhow!("recorded lock poisoned"))?
.push((provider_addr, bytes_delivered, amount));
Ok(())
}
fn pool_deposit(&self) -> Option<U256> {
read_deposit(&self.deposit).ok()
}
async fn recover_pool(&self, seen_deposit: U256) -> Result<Recovery> {
let now = read_deposit(&self.deposit)?;
if now > seen_deposit {
return Ok(Recovery::ToppedUp(now));
}
if self.working.is_zero() {
return Ok(Recovery::Unavailable);
}
let spent = {
let recorded = self
.recorded
.lock()
.map_err(|_| anyhow::anyhow!("recorded lock poisoned"))?;
let mut latest: HashMap<Address, U256> = HashMap::new();
for (provider, _, amount) in recorded.iter() {
latest.insert(*provider, *amount);
}
latest
.values()
.fold(U256::ZERO, |a, b| a.saturating_add(*b))
};
let mut deposit = self
.deposit
.lock()
.map_err(|_| anyhow::anyhow!("deposit lock poisoned"))?;
let additional = self.working.saturating_sub(deposit.saturating_sub(spent));
if additional.is_zero() {
return Ok(Recovery::ToppedUp(*deposit));
}
self.topups
.lock()
.map_err(|_| anyhow::anyhow!("topups lock poisoned"))?
.push((Address::ZERO, additional));
let added = if self.funds {
self.landing_cap
.map_or(additional, |cap| additional.min(cap))
} else {
U256::ZERO
};
if added.is_zero() {
return Ok(Recovery::Unavailable);
}
*deposit = deposit.saturating_add(added);
if self.ceiling_follows {
*self
.ceiling
.lock()
.map_err(|_| anyhow::anyhow!("ceiling lock poisoned"))? = *deposit;
}
Ok(Recovery::ToppedUp(*deposit))
}
}
#[allow(clippy::too_many_arguments)]
async fn serve_with_deposit_ceiling(
conn: Connection,
eth: &Arc<PrivateKeySigner>,
slash: &Eip712Domain,
payloads: &[Arc<Vec<u8>>],
rate: u64,
deposit: &SharedDeposit,
resume_delay: Duration,
initial_ceiling: U256,
lane: &Mutex<ScriptedLane>,
) -> Result<()> {
let (mut send, mut recv) = conn
.accept_bi()
.await
.map_err(|e| anyhow::anyhow!("accept_bi: {e}"))?;
let req = read_stream_request(&mut recv).await?;
let payload: &[u8] = payloads
.iter()
.find(|p| *Hash::new(p.as_slice()).as_bytes() == req.hash)
.map(|p| p.as_slice())
.ok_or_else(|| anyhow::anyhow!("deposit-capped upstream: unknown hash requested"))?;
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let (resp, resp_ext) = signed_response(&req, eth, slash, rate, total_bytes, None)?;
write_frame(&mut send, &encode_stream_response(&resp, Some(&resp_ext))?)
.await
.map_err(|e| anyhow::anyhow!("write response: {e}"))?;
if !resume_delay.is_zero() && read_deposit(deposit)? > initial_ceiling {
tokio::time::sleep(resume_delay).await;
}
let wire = honest_bao_wire_from(payload, req.byte_offset)?;
let frames = wire.len().div_ceil(WIRE_FRAME);
let mut wire_sent: u64 = 0;
let mut settled: u64 = 0;
let mut stream_root: Option<B256> = None;
for (ix, chunk) in wire.chunks(WIRE_FRAME).enumerate() {
write_frame(
&mut send,
&encode_message(&ClientMessage::ChunkData(ChunkData::new(chunk.to_vec())?))?,
)
.await
.map_err(|e| anyhow::anyhow!("write chunk: {e}"))?;
wire_sent = wire_sent.saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX));
let done = ix + 1 == frames;
while let Some(served) = payable_interval(wire_sent, settled, done) {
if !settle_voucher(
&mut send,
&mut recv,
deposit,
lane,
&mut stream_root,
served,
)
.await?
{
let _ = send.finish();
conn.closed().await;
return Ok(());
}
settled = settled.saturating_add(served);
}
}
write_frame(&mut send, &encode_message(&ClientMessage::StreamEnd)?)
.await
.map_err(|e| anyhow::anyhow!("write end: {e}"))?;
let _ = send.finish();
conn.closed().await;
Ok(())
}
#[expect(
clippy::print_stderr,
reason = "test harness diagnostic surfaced in the nextest log"
)]
async fn settle_voucher(
send: &mut iroh::endpoint::SendStream,
recv: &mut iroh::endpoint::RecvStream,
deposit: &SharedDeposit,
lane: &Mutex<ScriptedLane>,
stream_root: &mut Option<B256>,
served: u64,
) -> Result<bool> {
for _ in 0..4u8 {
let proof = read_proof(recv).await?;
if matches!(proof, ClientMessage::ChunkPreimage(_)) && stream_root.is_none() {
write_rejection(send, VoucherRejectReason::UnanchoredPreimage).await?;
return Ok(false);
}
let (advanced, credit, claim, dep) = {
let mut lane = lane
.lock()
.map_err(|_| anyhow::anyhow!("scripted lane lock poisoned"))?;
let mut candidate = *lane;
let advanced = candidate.apply(&proof, *stream_root);
let claim = candidate.claim();
let dep = read_deposit(deposit)?;
let credit = (claim <= dep).then(|| candidate.credit(&proof, advanced, served));
if matches!(credit, Some(Credit::Paid | Credit::Unpaid)) {
*lane = candidate;
}
(advanced, credit, claim, dep)
};
eprintln!(
"DIAG proof={} adv={advanced} credit={credit:?} claim={claim} dep={dep}",
match &proof {
ClientMessage::Voucher(v) =>
format!("V(amt={},root={:02x})", v.amount, v.chain_root[0]),
ClientMessage::ChunkPreimage(p) => format!("P(idx={})", p.index),
_ => "?".to_string(),
},
);
match credit {
None => {
write_rejection(send, VoucherRejectReason::SpendingCapExhausted).await?;
return Ok(false);
}
Some(Credit::Refused) => {
write_rejection(send, VoucherRejectReason::AmountRegression).await?;
return Ok(false);
}
Some(accepted) => {
if let ClientMessage::Voucher(v) = &proof {
let root = B256::from(v.chain_root);
*stream_root = (!root.is_zero()).then_some(root);
}
if accepted == Credit::Paid {
return Ok(true);
}
}
}
}
anyhow::bail!("deposit-capped upstream: four proofs in a row credited nothing")
}
async fn write_rejection(
send: &mut iroh::endpoint::SendStream,
reason: VoucherRejectReason,
) -> Result<()> {
write_frame(
send,
&encode_message(&ClientMessage::StreamError(StreamError::VoucherRejected {
reason,
bundle: None,
}))?,
)
.await
.map_err(|e| anyhow::anyhow!("write rejection: {e}"))
}
fn payable_interval(sent: u64, settled: u64, done: bool) -> Option<u64> {
let unpaid = sent.saturating_sub(settled);
if unpaid >= CHUNK_BYTES.saturating_add(decdn_client::VERIFY_LAG_BYTES) {
Some(CHUNK_BYTES)
} else if done && unpaid > 0 {
Some(unpaid.min(CHUNK_BYTES))
} else {
None
}
}
fn chunk_due(chunk_price: U256, served: u64) -> U256 {
let chunk = U256::from(CHUNK_BYTES);
(chunk_price.saturating_mul(U256::from(served)) + chunk - U256::from(1)) / chunk
}
#[allow(clippy::too_many_arguments)]
fn spawn_deposit_capped_server(
ep: iroh::Endpoint,
a_eth: Arc<PrivateKeySigner>,
slash: Eip712Domain,
payloads: Vec<Arc<Vec<u8>>>,
rate: u64,
deposit: SharedDeposit,
resume_delay: Duration,
initial_ceiling: U256,
) -> tokio::task::JoinHandle<()> {
let total_bytes = payloads
.first()
.map_or(0, |p| u64::try_from(p.len()).unwrap_or(u64::MAX));
let payloads = Arc::new(payloads);
let lane: Arc<Mutex<ScriptedLane>> = Arc::new(Mutex::new(ScriptedLane::default()));
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let eth = Arc::clone(&a_eth);
let dom = slash.clone();
let payloads = Arc::clone(&payloads);
let deposit = Arc::clone(&deposit);
let lane = Arc::clone(&lane);
if conn.alpn() == ALPN_PROBE {
tokio::spawn(async move {
let _ = answer_probe(conn, ð, &dom, rate, total_bytes).await;
});
} else {
tokio::spawn(async move {
let _ = serve_with_deposit_ceiling(
conn,
ð,
&dom,
&payloads,
rate,
&deposit,
resume_delay,
initial_ceiling,
&lane,
)
.await;
});
}
}
})
}
struct TopUpFixture {
origin: NodeOrigin,
engine: CacheEngine,
_engine_tmp: tempfile::TempDir,
opener: Arc<FundingOpener>,
metrics: Arc<Metrics>,
local_rep: Arc<LocalReputation>,
provider: iroh::PublicKey,
recorded: Arc<Mutex<Vec<ProgressEntry>>>,
ep_a: iroh::Endpoint,
ep_b: iroh::Endpoint,
task_a: tokio::task::JoinHandle<()>,
}
impl TopUpFixture {
async fn shutdown(self) -> Result<()> {
shutdown([self.task_a], [&self.ep_b, &self.ep_a]).await?;
Ok(())
}
}
#[derive(Debug, Clone, Copy)]
struct TopUpSetup {
initial_micro_usdc: u64,
working_micro_usdc: u64,
funds: bool,
landing_cap_micro_usdc: Option<u64>,
ceiling_micro_usdc: Option<u64>,
resume_delay: Duration,
}
impl TopUpSetup {
const fn honest(initial_micro_usdc: u64, working_micro_usdc: u64, funds: bool) -> Self {
Self {
initial_micro_usdc,
working_micro_usdc,
funds,
landing_cap_micro_usdc: None,
ceiling_micro_usdc: None,
resume_delay: Duration::ZERO,
}
}
}
async fn top_up_fixture(payload: Arc<Vec<u8>>, setup: TopUpSetup) -> Result<TopUpFixture> {
top_up_fixture_multi(vec![payload], setup).await
}
async fn top_up_fixture_multi(
payloads: Vec<Arc<Vec<u8>>>,
setup: TopUpSetup,
) -> Result<TopUpFixture> {
top_up_fixture_multi_rep(payloads, setup, LocalReputationConfig::default()).await
}
async fn top_up_fixture_multi_rep(
payloads: Vec<Arc<Vec<u8>>>,
setup: TopUpSetup,
rep_config: LocalReputationConfig,
) -> Result<TopUpFixture> {
let payload = Arc::clone(
payloads
.first()
.ok_or_else(|| anyhow::anyhow!("at least one payload"))?,
);
let hash = Hash::new(payload.as_ref());
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let deposit = shared_deposit(setup.initial_micro_usdc);
let ceiling = shared_deposit(setup.ceiling_micro_usdc.unwrap_or(setup.initial_micro_usdc));
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_deposit_capped_server(
ep_a.clone(),
Arc::clone(&a_eth),
slash_domain(),
payloads,
RATE,
Arc::clone(&ceiling),
setup.resume_delay,
U256::from(setup.ceiling_micro_usdc.unwrap_or(setup.initial_micro_usdc)),
);
let (ep_b, _) = local_endpoint(fresh_key(), vec![]).await?;
let b_id = fresh_key().public();
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(rep_config)?);
let metrics = Arc::new(Metrics::new());
let recorded = Arc::new(Mutex::new(Vec::new()));
let opener = Arc::new(FundingOpener {
pool_id: B256::repeat_byte(0x15),
deposit,
ceiling,
ceiling_follows: setup.ceiling_micro_usdc.is_none(),
signer: Arc::new(PrivateKeySigner::random()),
voucher_domain: voucher_dom(),
recorded: Arc::clone(&recorded),
topups: Arc::new(Mutex::new(Vec::new())),
working: U256::from(setup.working_micro_usdc),
funds: setup.funds,
landing_cap: setup.landing_cap_micro_usdc.map(U256::from),
});
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let (origin, engine, engine_tmp) = build_origin_with_probe_caches(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
hash,
Arc::clone(&opener) as Arc<dyn PoolOpener>,
&local_rep,
&metrics,
providers,
addr_map,
DEFAULT_TEST_PULL_DEADLINES.0,
DEFAULT_TEST_PULL_DEADLINES.1,
total_bytes.saturating_mul(4),
NegativeProbeCache::new(),
PositiveProbeCache::new(),
U256::ZERO,
)
.await;
Ok(TopUpFixture {
origin,
engine,
_engine_tmp: engine_tmp,
opener,
metrics,
local_rep,
provider: a_id,
recorded,
ep_a,
ep_b,
task_a,
})
}
fn topup_log(opener: &FundingOpener) -> Result<Vec<(Address, U256)>> {
Ok(opener
.topups
.lock()
.map_err(|_| anyhow::anyhow!("topups lock poisoned"))?
.clone())
}
fn multi_interval_payload() -> Arc<Vec<u8>> {
let len = usize::try_from(MB_BYTES).unwrap_or(usize::MAX) * 3 + 777;
Arc::new(
(0..len)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect(),
)
}
#[tokio::test(flavor = "multi_thread")]
async fn a_pull_larger_than_the_working_deposit_tops_up_once_and_completes() -> Result<()> {
let payload = multi_interval_payload();
let fixture = top_up_fixture(
Arc::clone(&payload),
TopUpSetup::honest(2 * RATE, 200 * RATE, true),
)
.await?;
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, Hash::new(payload.as_ref()), u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("the recovery top-up pull never finished"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::AlreadyAdmitted),
"a topped-up pull must deliver the blob, not NotFound"
);
let bytes = fixture.engine.get(Hash::new(payload.as_ref())).await?;
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"the resumed pull delivered {} bytes, expected {}",
bytes.len(),
payload.len()
);
let log = topup_log(&fixture.opener)?;
anyhow::ensure!(
log.len() == 1,
"exactly one recovery top-up should have funded this pull, got {log:?}"
);
anyhow::ensure!(
log.first().is_some_and(|(_, additional)| {
*additional >= U256::from(200 * RATE - 2 * RATE)
&& *additional <= U256::from(200 * RATE)
}),
"the top-up must raise spendable to the WORKING deposit, got {log:?}"
);
assert_counter(&fixture.metrics, "node_pull_recovery_step_total", 1)?;
assert_counter(&fixture.metrics, "node_pull_recovery_step_refused_total", 0)?;
assert_counter(&fixture.metrics, "node_pull_unreachable_total", 0)?;
assert_counter(&fixture.metrics, "node_pull_corruption_total", 0)?;
anyhow::ensure!(
fixture.local_rep.score(fixture.provider) > 0.5,
"an upstream that served through a top-up must be scored as delivering, got {}",
fixture.local_rep.score(fixture.provider)
);
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_slow_post_topup_delivery_scores_slow_not_instant() -> Result<()> {
const SLOW_RESUME: Duration = Duration::from_secs(3);
const REFERENCE_BPS: u64 = 1024 * 1024 * 1024;
let payload = multi_interval_payload();
let mut rep_config = LocalReputationConfig::default();
rep_config.alpha = 1.0;
rep_config.reference_bps = REFERENCE_BPS;
let mut setup = TopUpSetup::honest(2 * RATE, 200 * RATE, true);
setup.resume_delay = SLOW_RESUME;
let fixture = top_up_fixture_multi_rep(vec![Arc::clone(&payload)], setup, rep_config).await?;
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, Hash::new(payload.as_ref()), u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("the recovery top-up pull never finished"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::AlreadyAdmitted),
"a topped-up pull must deliver the blob, not NotFound"
);
let bytes = fixture.engine.get(Hash::new(payload.as_ref())).await?;
anyhow::ensure!(
bytes.as_ref() == payload.as_slice(),
"the resumed pull delivered {} bytes, expected {}",
bytes.len(),
payload.len()
);
assert_counter(&fixture.metrics, "node_pull_recovery_step_total", 1)?;
assert_counter(&fixture.metrics, "node_pull_corruption_total", 0)?;
assert_counter(&fixture.metrics, "node_pull_unreachable_total", 0)?;
let score = fixture.local_rep.score(fixture.provider);
anyhow::ensure!(
score > 0.6,
"a completed delivery must credit correctness+reachability, got {score}"
);
anyhow::ensure!(
score < 0.90,
"a slow post-top-up delivery must score slow; a score of {score} means the resume \
leg's transfer was wrongly excluded from `elapsed` (folded into `paid_wait`) — #1602"
);
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn the_resumed_leg_does_not_re_pay_for_delivered_bytes() -> Result<()> {
let payload = multi_interval_payload();
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let fixture = top_up_fixture(
Arc::clone(&payload),
TopUpSetup::honest(2 * RATE, 200 * RATE, true),
)
.await?;
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, Hash::new(payload.as_ref()), u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("the recovery top-up pull never finished"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::AlreadyAdmitted),
"the pull must have completed for its spend to mean anything"
);
let log = progress_log(&fixture.recorded)?;
let (_, billed_wire, _) = log
.last()
.copied()
.ok_or_else(|| anyhow::anyhow!("a paid pull must have persisted a watermark"))?;
let billed_wire = u64::try_from(billed_wire).unwrap_or(u64::MAX);
let whole_blob_wire = u64::try_from(honest_bao_wire_from(&payload, 0)?.len()).unwrap_or(0);
let proof_slack = whole_blob_wire.saturating_sub(total_bytes);
let ceiling = whole_blob_wire
.saturating_add(proof_slack)
.saturating_add(decdn_cache::CHUNK_GROUP_BYTES);
anyhow::ensure!(
billed_wire >= whole_blob_wire,
"UNDER-PAY: billed {billed_wire} wire bytes for a blob whose single encoding costs \
{whole_blob_wire}. Content was delivered that no voucher covered — resuming past the \
PAID frontier instead of at it"
);
anyhow::ensure!(
billed_wire <= ceiling,
"DOUBLE-PAY: billed {billed_wire} wire bytes, but one encoding costs {whole_blob_wire} \
and the resume slack allows at most {ceiling}. The resumed leg re-bought the prefix — \
which is what resuming from zero does"
);
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_node_crying_poverty_gets_one_recovery_step_and_no_more() -> Result<()> {
let payload = multi_interval_payload();
let fixture = top_up_fixture(
Arc::clone(&payload),
TopUpSetup {
initial_micro_usdc: 10_000 * RATE,
working_micro_usdc: 20_000 * RATE,
funds: true,
landing_cap_micro_usdc: None,
ceiling_micro_usdc: Some(0),
resume_delay: Duration::ZERO,
},
)
.await?;
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, Hash::new(payload.as_ref()), u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("the pull never ended"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a refused pull must not surface bytes"
);
let log = topup_log(&fixture.opener)?;
anyhow::ensure!(
log.len() <= 1,
"an upstream that verifies no byte must buy at most one recovery step, got {log:?}"
);
assert_counter(
&fixture.metrics,
"node_pull_recovery_step_total",
u64::try_from(log.len())?,
)?;
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_topup_that_adds_no_headroom_ends_the_pull() -> Result<()> {
let payload = multi_interval_payload();
let fixture = top_up_fixture(
Arc::clone(&payload),
TopUpSetup::honest(2 * RATE, 200 * RATE, false),
)
.await?;
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, Hash::new(payload.as_ref()), u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("an unfundable pull must not hang"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"an unfundable pull must not surface bytes"
);
let log = topup_log(&fixture.opener)?;
anyhow::ensure!(
log.len() == 1,
"the funding attempt must be made exactly once, then abandoned — got {log:?}"
);
assert_counter(&fixture.metrics, "node_pull_recovery_step_total", 0)?;
assert_counter(&fixture.metrics, "node_pull_recovery_step_refused_total", 1)?;
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_landing_short_of_the_owed_debt_ends_the_fill_after_one_step() -> Result<()> {
let payload = multi_interval_payload();
let hash = Hash::new(payload.as_ref());
let initial = 2 * RATE;
let cap = RATE;
let fixture = top_up_fixture(
Arc::clone(&payload),
TopUpSetup {
landing_cap_micro_usdc: Some(cap),
..TopUpSetup::honest(initial, 200 * RATE, true)
},
)
.await?;
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("an underfunded pull must not hang"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"a landing short of the owed debt ends the fill funding needed, got {got:?}"
);
let log = topup_log(&fixture.opener)?;
anyhow::ensure!(log.len() == 1, "exactly one step, got {log:?}");
assert_counter(&fixture.metrics, "node_pull_recovery_step_total", 1)?;
anyhow::ensure!(
read_deposit(&fixture.opener.deposit)? == U256::from(initial + cap),
"the one step raises the deposit by exactly what landed"
);
anyhow::ensure!(
fixture.engine.present_ranges(hash).await?.is_complete(),
"the fill verifies the whole blob, tail included, before the first step"
);
let paid = progress_log(&fixture.recorded)?
.iter()
.map(|(_, _, amount)| *amount)
.max()
.unwrap_or(U256::ZERO);
anyhow::ensure!(
paid == U256::from(3 * RATE),
"the lane must hold the three released MiB and leave the tail owed: paid {paid}"
);
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_short_landing_keeps_what_landed_and_completes_the_pull() -> Result<()> {
let payload = multi_interval_payload();
let initial = 2 * RATE;
let cap = 2 * RATE;
let fixture = top_up_fixture(
Arc::clone(&payload),
TopUpSetup {
landing_cap_micro_usdc: Some(cap),
..TopUpSetup::honest(initial, 200 * RATE, true)
},
)
.await?;
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, Hash::new(payload.as_ref()), u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("an underfunded pull must not hang"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::AlreadyAdmitted),
"the short landing must carry the pull to the end"
);
let log = topup_log(&fixture.opener)?;
anyhow::ensure!(!log.is_empty(), "the pull needs a landing, got none");
let steps = u64::try_from(log.len())?;
anyhow::ensure!(
read_deposit(&fixture.opener.deposit)? == U256::from(initial + cap * steps),
"each short landing must raise the deposit by exactly what landed"
);
let paid = progress_log(&fixture.recorded)?
.iter()
.map(|(_, _, amount)| *amount)
.max()
.unwrap_or(U256::ZERO);
anyhow::ensure!(
paid > U256::from(initial),
"the pull must spend the headroom the landing added: paid {paid}"
);
assert_counter(&fixture.metrics, "node_pull_recovery_step_total", steps)?;
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_full_pool_refused_by_a_stale_upstream_settles_then_is_served() -> Result<()> {
let payload = multi_interval_payload();
let fixture = top_up_fixture(
Arc::clone(&payload),
TopUpSetup {
ceiling_micro_usdc: Some(2 * RATE),
..TopUpSetup::honest(200 * RATE, 100 * RATE, true)
},
)
.await?;
let ceiling = Arc::clone(&fixture.opener.ceiling);
let recorded = Arc::clone(&fixture.recorded);
let catch_up = tokio::spawn(async move {
while recorded.lock().map_or(true, |log| log.is_empty()) {
tokio::time::sleep(Duration::from_millis(1)).await;
}
if let Ok(mut ceiling) = ceiling.lock() {
*ceiling = U256::from(200 * RATE);
}
});
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, Hash::new(payload.as_ref()), u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("a settling pull must not hang"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
catch_up.await?;
anyhow::ensure!(
matches!(got, OriginFetch::AlreadyAdmitted),
"the upstream serves once it sees the deposit"
);
anyhow::ensure!(
topup_log(&fixture.opener)?.is_empty(),
"a full pool escrows nothing"
);
assert_counter(&fixture.metrics, "node_pull_recovery_step_total", 1)?;
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_zero_working_deposit_never_funds_a_pull() -> Result<()> {
let payload = multi_interval_payload();
let fixture =
top_up_fixture(Arc::clone(&payload), TopUpSetup::honest(2 * RATE, 0, true)).await?;
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, Hash::new(payload.as_ref()), u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("the pull never ended"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::NotFound),
"with top-up disabled an exhausted pull must fail, not fund itself"
);
anyhow::ensure!(
topup_log(&fixture.opener)?.is_empty(),
"a zero working deposit must not escrow anything"
);
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn concurrent_pulls_resume_at_their_own_frontier_not_the_lanes() -> Result<()> {
let len = usize::try_from(CHUNK_BYTES).unwrap_or(usize::MAX) * 4 + 777;
let payload_a: Arc<Vec<u8>> = Arc::new(
(0..len)
.map(|i| u8::try_from(i % 249).unwrap_or(0))
.collect(),
);
let payload_b: Arc<Vec<u8>> = Arc::new(
(0..len)
.map(|i| u8::try_from(i % 247).unwrap_or(1))
.collect(),
);
let hash_a = Hash::new(payload_a.as_ref());
let hash_b = Hash::new(payload_b.as_ref());
anyhow::ensure!(hash_a != hash_b, "fixtures must be distinct blobs");
let fixture = top_up_fixture_multi(
vec![Arc::clone(&payload_a), Arc::clone(&payload_b)],
TopUpSetup::honest(6 * RATE, 400 * RATE, true),
)
.await?;
let (got_a, got_b) = tokio::time::timeout(Duration::from_mins(2), async {
tokio::join!(
Origin::fetch(&fixture.origin, hash_a, u64::MAX),
Origin::fetch(&fixture.origin, hash_b, u64::MAX),
)
})
.await
.map_err(|_| {
anyhow::anyhow!(
"concurrent top-up pulls never finished within 2m — likely stalled frontier or \
wedge; topups={:?} pending={} stalled={} timeout={}",
topup_log(&fixture.opener).unwrap_or_default(),
counter_value(&fixture.metrics, "node_pull_pool_open_pending_total").unwrap_or(0),
counter_value(&fixture.metrics, "node_pull_stalled_total").unwrap_or(0),
counter_value(&fixture.metrics, "node_pull_timeout_total").unwrap_or(0)
)
})?;
let got_a = got_a.map_err(|e| anyhow::anyhow!("fetch A: {e}"))?;
anyhow::ensure!(
matches!(&got_a, OriginFetch::AlreadyAdmitted),
"pull A returned {got_a:?} (expected AlreadyAdmitted); a frontier overshoot past the \
delivered bytes splices a gap and surfaces NotFound after the hash check, while a \
stalled/timeout budget expiry leaves topup_log empty — got topups={:?} stalled={} \
pending={}",
topup_log(&fixture.opener).unwrap_or_default(),
counter_value(&fixture.metrics, "node_pull_stalled_total").unwrap_or(0),
counter_value(&fixture.metrics, "node_pull_pool_open_pending_total").unwrap_or(0)
);
let bytes_a = fixture.engine.get(hash_a).await?;
let got_b = got_b.map_err(|e| anyhow::anyhow!("fetch B: {e}"))?;
anyhow::ensure!(
matches!(&got_b, OriginFetch::AlreadyAdmitted),
"pull B returned {got_b:?} (expected AlreadyAdmitted); a frontier overshoot past the \
delivered bytes splices a gap and surfaces NotFound after the hash check, while a \
stalled/timeout budget expiry leaves topup_log empty — got topups={:?} stalled={} \
pending={}",
topup_log(&fixture.opener).unwrap_or_default(),
counter_value(&fixture.metrics, "node_pull_stalled_total").unwrap_or(0),
counter_value(&fixture.metrics, "node_pull_pool_open_pending_total").unwrap_or(0)
);
let bytes_b = fixture.engine.get(hash_b).await?;
anyhow::ensure!(bytes_a.as_ref() == payload_a.as_slice(), "pull A corrupted");
anyhow::ensure!(bytes_b.as_ref() == payload_b.as_slice(), "pull B corrupted");
anyhow::ensure!(
!topup_log(&fixture.opener)?.is_empty(),
"the initial deposit was never exhausted; the test no longer covers the resume path"
);
fixture.shutdown().await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_long_pull_recovers_again_after_verified_progress() -> Result<()> {
let len = usize::try_from(MB_BYTES).unwrap_or(usize::MAX) * 9 + 777;
let payload: Arc<Vec<u8>> = Arc::new(
(0..len)
.map(|i| u8::try_from(i % 253).unwrap_or(0))
.collect(),
);
let fixture = top_up_fixture(
Arc::clone(&payload),
TopUpSetup::honest(2 * RATE, 3 * RATE, true),
)
.await?;
let got = tokio::time::timeout(
Duration::from_mins(1),
Origin::fetch(&fixture.origin, Hash::new(payload.as_ref()), u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("a pull funded step by step must not loop forever"))?
.map_err(|e| anyhow::anyhow!("fetch: {e}"))?;
anyhow::ensure!(
matches!(got, OriginFetch::AlreadyAdmitted),
"each step paid for new bytes, so the steps carry the pull to the end"
);
let log = topup_log(&fixture.opener)?;
anyhow::ensure!(
log.len() >= 2,
"a blob past one working deposit needs more than one step, got {log:?}"
);
assert_counter(
&fixture.metrics,
"node_pull_recovery_step_total",
u64::try_from(log.len())?,
)?;
fixture.shutdown().await?;
Ok(())
}
#[allow(clippy::too_many_arguments, clippy::expect_used)]
async fn build_origin_economics(
ep_b: &iroh::Endpoint,
b_dht: DhtNodeId,
buyer: Arc<dyn PoolOpener>,
local_rep: &Arc<LocalReputation>,
metrics: &Arc<Metrics>,
providers: Vec<DhtNodeId>,
addr_map: HashMap<DhtNodeId, Address>,
serve_economics: Arc<dyn decdn_node::serve_economics::ServeEconomicsPolicy>,
operator_bps: u16,
sell_rate_base: u64,
warming: Arc<decdn_node::warming_allowance::WarmingAllowance>,
frequency_estimator: Option<Arc<dyn decdn_cache::FrequencyEstimator>>,
) -> (NodeOrigin, CacheEngine, tempfile::TempDir) {
let stakers = ConfigStakerSet::new(providers.iter().copied().collect());
let mut dir = HashMap::new();
dir.insert(U256::ZERO, providers);
let (engine, engine_tmp) = throwaway_engine()
.await
.expect("throwaway engine for the economics fixture");
let origin = NodeOrigin::new();
origin.provision(NodeOriginDeps {
endpoint: ep_b.clone(),
dial_runtime: tokio::runtime::Handle::current(),
routing_table: Arc::new(Mutex::new(RoutingTable::new(b_dht))),
staker_set: Arc::new(stakers) as Arc<dyn StakerSet>,
origin_directory: Arc::new(StaticOriginDirectory::new(dir)) as Arc<dyn OriginDirectory>,
addr_resolver: Arc::new(StaticNodeAddressDirectory::new(addr_map))
as Arc<dyn NodeAddressResolver>,
buyer,
self_id: b_dht,
slash_domain: slash_domain(),
bind_domain: binding_dom(),
local_rep: Arc::clone(local_rep),
negative_cache: NegativeProbeCache::new(),
probe_cache: PositiveProbeCache::new(),
metrics: Arc::clone(metrics),
registry_regions: Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())),
config: NodeOriginConfig {
probe_fanout: 5,
pull_timeout: DEFAULT_TEST_PULL_DEADLINES.0,
stall_window: DEFAULT_TEST_PULL_DEADLINES.1,
min_throughput_bps: 0,
max_blob_size_bytes: 0,
max_rate_per_mb: 0,
event_poll_interval: Duration::from_millis(50),
lookup: decdn_node::dht::LookupConfig::default(),
own_region: None,
serve_economics,
operator_shares: decdn_node::fee_shares::OperatorShares::new(operator_bps),
frequency_estimator,
sell_rate_base,
warming,
},
ledgers: Arc::new(decdn_node::buyer_ledgers::BuyerLedgers::default()),
wedged_providers: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
engine: engine.clone(),
});
(origin, engine, engine_tmp)
}
#[derive(Debug)]
struct FixedHeat(u32);
impl decdn_cache::FrequencyEstimator for FixedHeat {
fn observe(&self, _hash: Hash) {}
fn estimate(&self, _hash: Hash) -> u32 {
self.0
}
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)]
async fn attack_a_over_market_loss_is_bounded() -> Result<()> {
const N_MAX: u32 = 64;
const DISCOUNT_BPS: u32 = 5000;
const HEAT: u32 = 128;
const OP_BPS: u16 = 6000;
const SELL: u64 = 25;
const QUOTE_A: u64 = (OP_BPS as u64) * (N_MAX as u64) * SELL / 10_000;
const LEDGER_DEPOSIT: u64 = 1_000_000_000;
let payload = vec![0xADu8; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_sk = fresh_key();
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let ab_pool_id = B256::repeat_byte(0xA9);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
ab_pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(LEDGER_DEPOSIT),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let a_metrics = Arc::new(Metrics::new());
let a_limiter = permissive_limiter(&a_metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&a_metrics,
a_limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
QUOTE_A,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
QUOTE_A,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, addr_b) = local_endpoint(b_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let warming = Arc::new(decdn_node::warming_allowance::WarmingAllowance::new(
LEDGER_DEPOSIT,
0,
));
let serve_economics = Arc::new(decdn_node::serve_economics::MarginPolicy::new(
DISCOUNT_BPS,
N_MAX,
)) as Arc<dyn decdn_node::serve_economics::ServeEconomicsPolicy>;
let recorded: Arc<Mutex<Vec<ProgressEntry>>> = Arc::new(Mutex::new(Vec::new()));
let buyer = Arc::new(StubOpener {
pool_id: ab_pool_id,
deposit: U256::from(LEDGER_DEPOSIT),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::clone(&recorded),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_economics(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
serve_economics,
OP_BPS,
SELL,
Arc::clone(&warming),
Some(Arc::new(FixedHeat(HEAT)) as Arc<dyn decdn_cache::FrequencyEstimator>),
)
.await;
let bought = tokio::time::timeout(
Duration::from_secs(20),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("origin fetch never returned"))?
.map_err(|e| anyhow::anyhow!("origin fetch: {e}"))?;
anyhow::ensure!(
matches!(bought, OriginFetch::AlreadyAdmitted),
"the attack quotes AT the ceiling; the gate must admit it, got {bought:?}"
);
anyhow::ensure!(
engine.get(hash).await?.as_ref() == payload.as_slice(),
"B's cache must hold the bought bytes"
);
let b_eth = Arc::new(PrivateKeySigner::random());
let client_signer = Arc::new(PrivateKeySigner::random());
let b_pool_id = B256::repeat_byte(0xB9);
let store_b = Arc::new(MemoryPoolStateStore::new());
let b_lane = decdn_incentive::LaneKey {
pool_id: b_pool_id,
signer: client_signer.address(),
provider: b_eth.address(),
};
store_b.record(&LaneState::hydrate(
b_pool_id,
client_signer.address(),
b_eth.address(),
U256::from(LEDGER_DEPOSIT),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let handler_b = build_handler_full_configured(
b_id,
&b_eth,
&b_metrics,
permissive_limiter(&b_metrics),
engine,
store_b.clone() as Arc<dyn PoolStateStore>,
SELL,
&domains,
16,
|deps| {
deps.warming_credit = Arc::new(
decdn_node::warming_allowance::DirectWarmingCreditSink::new(Arc::clone(&warming)),
);
deps.operator_shares = decdn_node::fee_shares::OperatorShares::new(OP_BPS);
},
)?;
let task_b = spawn_server(ep_b.clone(), handler_b);
let client_sk = fresh_key();
let client_id = client_sk.public();
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target_b = EndpointAddr::new(b_id).with_ip_addr(addr_b);
let binding = decdn_client::sign_client_binding(
&client_signer,
B256::from(*client_id.as_bytes()),
&binding_dom(),
)?;
let ctx_client_to_b = PoolContext {
pool_id: b_pool_id,
provider: b_eth.address(),
deposit: U256::from(LEDGER_DEPOSIT),
client_signer: Arc::clone(&client_signer),
voucher_domain: voucher_dom(),
prior_bytes_delivered: U256::ZERO,
prior_amount: U256::ZERO,
client_binding: Some(binding),
capability: None,
};
let got = decdn_client::stream_fetch(
&client_ep,
target_b,
&ctx_client_to_b,
&slash_domain(),
b_eth.address(),
*hash.as_bytes(),
0,
0x00a1_0001,
Duration::from_secs(20),
)
.await?;
anyhow::ensure!(got.as_ref() == payload.as_slice(), "resale bytes mismatch");
let paid_out = progress_log(&recorded)?
.last()
.copied()
.map(|(_, _, amount)| amount)
.ok_or_else(|| anyhow::anyhow!("B never recorded a payment to A"))?;
let paid_in = store_b
.get(b_lane)?
.ok_or_else(|| anyhow::anyhow!("B<->client lane vanished"))?
.last_amount();
let margin_recovered = paid_in.saturating_mul(U256::from(OP_BPS)) / U256::from(10_000u64);
let loss = paid_out.saturating_sub(margin_recovered);
let wire = support::bao_wire_len_whole(total_bytes);
let mb_wire = wire.div_ceil(MB_BYTES);
let bound = U256::from(OP_BPS) * U256::from(SELL) * U256::from(u64::from(N_MAX) - 1)
/ U256::from(10_000u64)
* U256::from(mb_wire);
anyhow::ensure!(
loss <= bound,
"Attack A loss bound violated: paid_out {paid_out}, paid_in {paid_in}, margin \
recovered {margin_recovered}, measured loss {loss} > bound {bound} \
(op_bps/10_000 * sell * (n_max-1) * {mb_wire} MB)"
);
anyhow::ensure!(
loss > U256::ZERO,
"the attack must be genuinely lossy for the bound above to mean anything, \
got paid_out {paid_out}, margin_recovered {margin_recovered}"
);
shutdown([task_a, task_b], [&client_ep, &ep_b, &ep_a]).await?;
Ok(())
}
#[allow(
clippy::too_many_arguments,
clippy::expect_used,
clippy::too_many_lines
)]
async fn attack_b_attempt(
a_sk: iroh::SecretKey,
salt: u8,
quote: u64,
sell: u64,
op_bps: u16,
discount_bps: u32,
n_max: u32,
warming: &Arc<decdn_node::warming_allowance::WarmingAllowance>,
serves: u32,
) -> Result<(bool, Arc<Metrics>)> {
const LEDGER_DEPOSIT: u64 = 1_000_000_000;
let payload = vec![salt; PAYLOAD_LEN];
let hash = Hash::new(&payload);
let total_bytes = u64::try_from(PAYLOAD_LEN).unwrap_or(u64::MAX);
let (cache_a, hash_a, _tmp_a) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_a == hash, "fixture hash mismatch");
let a_id = a_sk.public();
let a_eth = Arc::new(PrivateKeySigner::random());
let b_buyer = Arc::new(PrivateKeySigner::random());
let ab_pool_id = B256::repeat_byte(salt);
let store_a = Arc::new(MemoryPoolStateStore::new());
store_a.record(&LaneState::hydrate(
ab_pool_id,
b_buyer.address(),
a_eth.address(),
U256::from(LEDGER_DEPOSIT),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let a_metrics = Arc::new(Metrics::new());
let a_limiter = permissive_limiter(&a_metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_a = build_handler_full(
a_id,
&a_eth,
&a_metrics,
a_limiter,
cache_a,
store_a as Arc<dyn PoolStateStore>,
quote,
&domains,
16,
)?;
let (ep_a, addr_a) =
local_endpoint(a_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task_a = spawn_a_server(
ep_a.clone(),
handler_a,
Arc::clone(&a_eth),
slash_domain(),
total_bytes,
quote,
);
let b_sk = fresh_key();
let b_id = b_sk.public();
let (ep_b, addr_b) = local_endpoint(b_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let _ = probe_once(
&ep_b,
EndpointAddr::new(a_id).with_ip_addr(addr_a),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let b_metrics = Arc::new(Metrics::new());
let (providers, addr_map) =
one_provider(DhtNodeId::from_bytes(*a_id.as_bytes()), a_eth.address());
let serve_economics = Arc::new(decdn_node::serve_economics::MarginPolicy::new(
discount_bps,
n_max,
)) as Arc<dyn decdn_node::serve_economics::ServeEconomicsPolicy>;
let buyer = Arc::new(StubOpener {
pool_id: ab_pool_id,
deposit: U256::from(LEDGER_DEPOSIT),
signer: Arc::clone(&b_buyer),
voucher_domain: voucher_dom(),
recorded: Arc::new(Mutex::new(Vec::new())),
}) as Arc<dyn PoolOpener>;
let (origin, engine, _engine_tmp) = build_origin_economics(
&ep_b,
DhtNodeId::from_bytes(*b_id.as_bytes()),
buyer,
&local_rep,
&b_metrics,
providers,
addr_map,
serve_economics,
op_bps,
sell,
Arc::clone(warming),
None,
)
.await;
let bought = tokio::time::timeout(
Duration::from_secs(20),
Origin::fetch(&origin, hash, u64::MAX),
)
.await
.map_err(|_| anyhow::anyhow!("origin fetch never returned"))?
.map_err(|e| anyhow::anyhow!("origin fetch: {e}"))?;
let admitted = matches!(bought, OriginFetch::AlreadyAdmitted);
if admitted && serves > 0 {
let b_eth = Arc::new(PrivateKeySigner::random());
let store_b = Arc::new(MemoryPoolStateStore::new());
let mut lanes = Vec::new();
for i in 0..serves {
let client_signer = Arc::new(PrivateKeySigner::random());
let mut pool_id_bytes = [0u8; 32];
pool_id_bytes[0] = salt;
pool_id_bytes[1] = u8::try_from(i).unwrap_or(0xFF);
let pool_id = B256::from(pool_id_bytes);
store_b.record(&LaneState::hydrate(
pool_id,
client_signer.address(),
b_eth.address(),
U256::from(LEDGER_DEPOSIT),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
lanes.push((pool_id, client_signer));
}
let handler_b = build_handler_full_configured(
b_id,
&b_eth,
&b_metrics,
permissive_limiter(&b_metrics),
engine,
store_b as Arc<dyn PoolStateStore>,
sell,
&domains,
16,
|deps| {
deps.warming_credit =
Arc::new(decdn_node::warming_allowance::DirectWarmingCreditSink::new(
Arc::clone(warming),
));
deps.operator_shares = decdn_node::fee_shares::OperatorShares::new(op_bps);
},
)?;
let task_b = spawn_server(ep_b.clone(), handler_b);
for (i, (pool_id, client_signer)) in lanes.into_iter().enumerate() {
let client_sk = fresh_key();
let client_id = client_sk.public();
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target_b = EndpointAddr::new(b_id).with_ip_addr(addr_b);
let binding = decdn_client::sign_client_binding(
&client_signer,
B256::from(*client_id.as_bytes()),
&binding_dom(),
)?;
let ctx = PoolContext {
pool_id,
provider: b_eth.address(),
deposit: U256::from(LEDGER_DEPOSIT),
client_signer: Arc::clone(&client_signer),
voucher_domain: voucher_dom(),
prior_bytes_delivered: U256::ZERO,
prior_amount: U256::ZERO,
client_binding: Some(binding),
capability: None,
};
let got = decdn_client::stream_fetch(
&client_ep,
target_b,
&ctx,
&slash_domain(),
b_eth.address(),
*hash.as_bytes(),
0,
0x00b0_0000u64
.saturating_add(u64::from(salt) * 0x100)
.saturating_add(i as u64),
Duration::from_secs(20),
)
.await?;
anyhow::ensure!(
got.as_ref() == payload.as_slice(),
"resale #{i} bytes mismatch"
);
shutdown([], [&client_ep]).await?;
}
shutdown([task_a, task_b], [&ep_b, &ep_a]).await?;
return Ok((admitted, b_metrics));
}
shutdown([task_a], [&ep_b, &ep_a]).await?;
Ok((admitted, b_metrics))
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::expect_used, clippy::too_many_lines)]
async fn attack_b_dud_flood_is_bounded_and_vindication_works() -> Result<()> {
const N_MAX: u32 = 64;
const DISCOUNT_BPS: u32 = 5000;
const OP_BPS: u16 = 6000;
const SELL: u64 = 1000;
const QUOTE_MARKET: u64 = 1000;
const BUDGET: u64 = 2000;
let warming = Arc::new(decdn_node::warming_allowance::WarmingAllowance::new(
BUDGET, 0,
));
let src1_sk = fresh_key();
let src1_id = decdn_node::warming_allowance::SourceId::from_bytes(*src1_sk.public().as_bytes());
let (first_admitted, _) = attack_b_attempt(
src1_sk.clone(),
0xB1,
QUOTE_MARKET,
SELL,
OP_BPS,
DISCOUNT_BPS,
N_MAX,
&warming,
1,
)
.await?;
anyhow::ensure!(
first_admitted,
"the first (dud) at-market buy from a fresh source must be admitted"
);
anyhow::ensure!(
warming.available(src1_id),
"one dud (bought at market, served once) costs only its fee skim; the budget \
covers it and the source keeps warming"
);
let (second_dud_admitted, _) = attack_b_attempt(
src1_sk.clone(),
0xB2,
QUOTE_MARKET,
SELL,
OP_BPS,
DISCOUNT_BPS,
N_MAX,
&warming,
0,
)
.await?;
anyhow::ensure!(
second_dud_admitted,
"a dud while the source's allowance is still positive must be admitted"
);
anyhow::ensure!(
!warming.available(src1_id),
"the duds must exhaust the budget and cut the source off"
);
for salt in [0xB3u8, 0xB4u8] {
let (flood_admitted, flood_metrics) = attack_b_attempt(
src1_sk.clone(),
salt,
QUOTE_MARKET,
SELL,
OP_BPS,
DISCOUNT_BPS,
N_MAX,
&warming,
0,
)
.await?;
anyhow::ensure!(
!flood_admitted,
"a further at-market cold buy from a spent source must refuse (salt {salt:#x})"
);
assert_counter(&flood_metrics, "serve_economics_refused_total", 1)?;
}
let src2_sk = fresh_key();
let src2_id = decdn_node::warming_allowance::SourceId::from_bytes(*src2_sk.public().as_bytes());
anyhow::ensure!(
warming.available(src2_id),
"an independent source must read available before it is ever touched"
);
let (second_admitted, _) = attack_b_attempt(
src2_sk,
0xC1,
QUOTE_MARKET,
SELL,
OP_BPS,
DISCOUNT_BPS,
N_MAX,
&warming,
0,
)
.await?;
anyhow::ensure!(
second_admitted,
"a second source's allowance must be untouched by another source's flood"
);
let src3_sk = fresh_key();
let src3_id = decdn_node::warming_allowance::SourceId::from_bytes(*src3_sk.public().as_bytes());
warming.debit_speculative(
src3_id,
decdn_cache::Hash::from_bytes([0xD0u8; 32]),
BUDGET - 500,
);
anyhow::ensure!(
warming.available(src3_id),
"precondition: the earlier duds leave the source warm"
);
let (vindicated_admitted, _) = attack_b_attempt(
src3_sk,
0xD1,
QUOTE_MARKET,
SELL,
OP_BPS,
DISCOUNT_BPS,
N_MAX,
&warming,
2,
)
.await?;
anyhow::ensure!(vindicated_admitted, "the vindication buy must be admitted");
anyhow::ensure!(
warming.available(src3_id),
"a blob re-served twice must refund its buy and keep the source warming"
);
Ok(())
}
fn two_providers(
a_dht: DhtNodeId,
a_eth: Address,
b_dht: DhtNodeId,
b_eth: Address,
) -> (Vec<DhtNodeId>, HashMap<DhtNodeId, Address>) {
let mut addr_map = HashMap::new();
addr_map.insert(a_dht, a_eth);
addr_map.insert(b_dht, b_eth);
(vec![a_dht, b_dht], addr_map)
}
async fn spawn_partial_holder(
payload: &[u8],
ab_pool_id: B256,
s_buyer_addr: Address,
coverage: Coverage,
refuse_client_conns: usize,
) -> Result<(
iroh::PublicKey,
std::net::SocketAddr,
Address,
iroh::Endpoint,
tokio::task::JoinHandle<()>,
Arc<MemoryPoolStateStore>,
)> {
let hash = Hash::new(payload);
let (cache, hash_h, tmp) = cache_with_blob(payload).await?;
anyhow::ensure!(hash_h == hash, "holder fixture hash mismatch");
std::mem::forget(tmp);
let (id, addr, eth, ep, task, store, _metrics) = spawn_holder_on(
cache,
payload,
ab_pool_id,
s_buyer_addr,
coverage,
refuse_client_conns,
)
.await?;
Ok((id, addr, eth, ep, task, store))
}
async fn spawn_block_holder(
payload: &[u8],
ab_pool_id: B256,
s_buyer_addr: Address,
blocks: &[u32],
) -> Result<(
iroh::PublicKey,
std::net::SocketAddr,
Address,
iroh::Endpoint,
tokio::task::JoinHandle<()>,
Arc<MemoryPoolStateStore>,
Arc<Metrics>,
)> {
let hash = Hash::new(payload);
let total = u64::try_from(payload.len())?;
let block = decdn_protocol::discovery_block_bytes();
let outboard = bytes::Bytes::from(
bao_tree::io::outboard::PreOrderMemOutboard::create(
payload,
decdn_cache::range_pull::IROH_BLOCK_SIZE,
)
.data,
);
let cache_tmp = tempfile::tempdir()?;
let cache = CacheEngine::open(cache_tmp.path(), vec![], 16).await?;
std::mem::forget(cache_tmp);
for &b in blocks {
let start = u64::from(b)
.checked_mul(block)
.filter(|&start| start < total)
.ok_or_else(|| anyhow::anyhow!("block {b} lies past the {total}-byte blob"))?;
let aligned = decdn_cache::range_pull::align_range(start, block.min(total - start), total)?;
let data = payload
.get(usize::try_from(aligned.fetch_start())?..usize::try_from(aligned.fetch_end())?)
.ok_or_else(|| anyhow::anyhow!("held block out of bounds"))?;
let bao = decdn_cache::range_pull::encode_verified_range(
*hash.as_bytes(),
&aligned,
data,
outboard.clone(),
)?;
cache
.admit_bao(hash, aligned.chunk_ranges().clone(), bao)
.await?;
}
let coverage =
Coverage::from_block_indices(decdn_protocol::num_blocks(total), blocks.iter().copied());
spawn_holder_on(cache, payload, ab_pool_id, s_buyer_addr, coverage, 0).await
}
async fn spawn_holder_on(
cache: CacheEngine,
payload: &[u8],
ab_pool_id: B256,
s_buyer_addr: Address,
coverage: Coverage,
refuse_client_conns: usize,
) -> Result<(
iroh::PublicKey,
std::net::SocketAddr,
Address,
iroh::Endpoint,
tokio::task::JoinHandle<()>,
Arc<MemoryPoolStateStore>,
Arc<Metrics>,
)> {
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let sk = fresh_key();
let id = sk.public();
let eth = Arc::new(PrivateKeySigner::random());
let store = Arc::new(MemoryPoolStateStore::new());
store.record(&LaneState::hydrate(
ab_pool_id,
s_buyer_addr,
eth.address(),
U256::from(DEPOSIT_MICRO_USDC),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler = build_handler_full(
id,
ð,
&metrics,
limiter,
cache,
store.clone() as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
)?;
let holder_metrics = Arc::clone(&metrics);
let (ep, addr) = local_endpoint(sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let task = {
use iroh::protocol::ProtocolHandler;
let ep = ep.clone();
let eth = Arc::clone(ð);
let slash = slash_domain();
tokio::spawn(async move {
let mut client_conns = 0usize;
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
if conn.alpn() == ALPN_CLIENT {
let nth = client_conns;
client_conns += 1;
if nth < refuse_client_conns {
let eth = Arc::clone(ð);
let dom = slash.clone();
tokio::spawn(async move {
let _ = serve_refusal(
conn,
ð,
&dom,
total_bytes,
RATE,
StreamError::NotFound,
)
.await;
});
continue;
}
}
if conn.alpn() == ALPN_PROBE {
let eth = Arc::clone(ð);
let dom = slash.clone();
let coverage = coverage.clone();
tokio::spawn(async move {
let _ = answer_probe_with_coverage(
conn,
ð,
&dom,
RATE,
total_bytes,
coverage,
)
.await;
});
} else {
let handler = Arc::clone(&handler);
tokio::spawn(async move {
let _ = decdn_node::handlers::client::ClientProtocol::new(handler)
.accept(conn)
.await;
});
}
}
})
};
Ok((id, addr, eth.address(), ep, task, store, holder_metrics))
}
#[allow(clippy::too_many_arguments)]
async fn build_serving_node(
hash: Hash,
ab_pool_id: B256,
s_buyer: &Arc<PrivateKeySigner>,
discovery: FixtureDiscovery,
probe_cache: PositiveProbeCache,
addr_map: HashMap<DhtNodeId, Address>,
holder_dials: &[(iroh::PublicKey, std::net::SocketAddr)],
leaves: &[(B256, Address)],
leaf_deposit: U256,
) -> Result<(
Arc<decdn_node::handlers::client::ClientHandler>,
EndpointAddr,
iroh::Endpoint,
Arc<Mutex<Vec<ProgressEntry>>>,
decdn_cache::CacheEngine,
Address,
Arc<Metrics>,
Arc<NodeOrigin>,
)> {
let s_sk = fresh_key();
let s_id = s_sk.public();
let s_eth = Arc::new(PrivateKeySigner::random());
let (ep_s, addr_s) = local_endpoint(s_sk, vec![ALPN_CLIENT.to_vec()]).await?;
for (holder_id, holder_addr) in holder_dials {
let _ = probe_once(
&ep_s,
EndpointAddr::new(*holder_id).with_ip_addr(*holder_addr),
*hash.as_bytes(),
1,
Duration::from_secs(10),
)
.await?;
}
let local_rep = Arc::new(LocalReputation::new(LocalReputationConfig::default())?);
let s_metrics = Arc::new(Metrics::new());
let (buyer, recorded) = stub_opener(ab_pool_id, s_buyer);
let (origin, _engine, _engine_tmp) = build_origin_with_discovery(
&ep_s,
DhtNodeId::from_bytes(*s_id.as_bytes()),
buyer,
&local_rep,
&s_metrics,
discovery,
addr_map,
DEFAULT_TEST_PULL_DEADLINES.0,
DEFAULT_TEST_PULL_DEADLINES.1,
0, NegativeProbeCache::new(),
probe_cache,
)
.await;
let origin_s = Arc::new(origin);
let cache_tmp = tempfile::tempdir()?;
let cache_s = decdn_cache::CacheEngine::open(cache_tmp.path(), vec![], 64).await?;
let cache_handle = cache_s.clone();
std::mem::forget(cache_tmp);
let store_s = Arc::new(MemoryPoolStateStore::new());
let mut pool_status_map: HashMap<B256, decdn_node::pool_view::PoolStatus> = HashMap::new();
for &(leaf_pool_id, leaf_eth_addr) in leaves {
store_s.record(&LaneState::hydrate(
leaf_pool_id,
leaf_eth_addr,
s_eth.address(),
leaf_deposit,
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
pool_status_map.insert(
leaf_pool_id,
decdn_node::pool_view::PoolStatus {
owner: leaf_eth_addr,
remaining: leaf_deposit,
lifecycle: decdn_node::pool_view::Lifecycle::Open,
},
);
}
let pool_view = Arc::new(StubPoolView {
status: pool_status_map,
}) as Arc<dyn decdn_node::pool_view::PoolView>;
let limiter = permissive_limiter(&s_metrics);
let domains = HandlerDomains {
slash: slash_domain(),
voucher: voucher_dom(),
binding: binding_dom(),
};
let handler_s = build_handler_full_configured(
s_id,
&s_eth,
&s_metrics,
limiter,
cache_s,
store_s as Arc<dyn PoolStateStore>,
RATE,
&domains,
16,
|deps| {
deps.pull_through = Some(Duration::from_secs(20));
deps.pull_through_origin = Some(Arc::clone(&origin_s));
deps.pool_view = Some(pool_view);
},
)?;
let target = EndpointAddr::new(s_id).with_ip_addr(addr_s);
Ok((
handler_s,
target,
ep_s,
recorded,
cache_handle,
s_eth.address(),
s_metrics,
origin_s,
))
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn two_partial_holders_assemble_over_the_real_paid_path() -> Result<()> {
let block: u64 = 16 * 1024;
let _block_guard = decdn_protocol::override_discovery_block_bytes_for_test(block);
anyhow::ensure!(
decdn_protocol::discovery_block_bytes() == block,
"override did not take"
);
let payload_len = usize::try_from(2 * block).unwrap_or(usize::MAX);
let payload: Vec<u8> = (0..payload_len)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect();
let hash = Hash::new(&payload);
let total_bytes = 2 * block;
anyhow::ensure!(
decdn_protocol::num_blocks(total_bytes) == 2,
"the blob must span exactly two discovery blocks under the override"
);
let ab_pool_id = B256::repeat_byte(0xA1);
let s_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a, store_a) = spawn_partial_holder(
&payload,
ab_pool_id,
s_buyer.address(),
Coverage::from_block_indices(2, [0].into_iter()),
0,
)
.await?;
let (b_id, b_addr, b_eth, ep_b, task_b, store_b) = spawn_partial_holder(
&payload,
ab_pool_id,
s_buyer.address(),
Coverage::from_block_indices(2, [1].into_iter()),
0,
)
.await?;
let (providers, addr_map) = two_providers(
DhtNodeId::from_bytes(*a_id.as_bytes()),
a_eth,
DhtNodeId::from_bytes(*b_id.as_bytes()),
b_eth,
);
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x1F);
let (handler_s, s_target, ep_s, recorded, cache_s, s_operator, s_metrics, _) =
build_serving_node(
hash,
ab_pool_id,
&s_buyer,
FixtureDiscovery::directory_only(providers, U256::ZERO),
PositiveProbeCache::new(),
addr_map,
&[(a_id, a_addr), (b_id, b_addr)],
&[(leaf_pool_id, leaf_eth.address())],
U256::from(DEPOSIT_MICRO_USDC),
)
.await?;
let task_s = spawn_server(ep_s.clone(), handler_s);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let outcome = leaf_paced_pull(
&leaf_ep,
s_target,
leaf_node_id,
&leaf_eth,
s_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await?;
anyhow::ensure!(outcome.completed, "leaf delivery did not complete");
anyhow::ensure!(outcome.hash_ok, "leaf received bytes failed the hash check");
anyhow::ensure!(
outcome.received == total_bytes,
"leaf received {} of {total_bytes} bytes",
outcome.received
);
anyhow::ensure!(
cache_s.has(hash).await?,
"S must promote the assembled blob on a complete delivery"
);
let lane_a = LaneKey {
pool_id: ab_pool_id,
signer: s_buyer.address(),
provider: a_eth,
};
let lane_b = LaneKey {
pool_id: ab_pool_id,
signer: s_buyer.address(),
provider: b_eth,
};
let a_delivered = store_a
.get(lane_a)?
.ok_or_else(|| anyhow::anyhow!("holder A lane vanished"))?
.last_bytes_delivered();
let b_delivered = store_b
.get(lane_b)?
.ok_or_else(|| anyhow::anyhow!("holder B lane vanished"))?
.last_bytes_delivered();
let a_wire =
u64::try_from(honest_bao_wire_range(&payload, 0, block)?.len()).unwrap_or(u64::MAX);
let b_wire =
u64::try_from(honest_bao_wire_range(&payload, block, 0)?.len()).unwrap_or(u64::MAX);
let whole_wire =
u64::try_from(honest_bao_wire_range(&payload, 0, 0)?.len()).unwrap_or(u64::MAX);
anyhow::ensure!(
a_delivered == U256::from(a_wire),
"holder A must be paid for exactly block 0's wire ({a_wire}), got {a_delivered}"
);
anyhow::ensure!(
b_delivered == U256::from(b_wire),
"holder B must be paid for exactly block 1's wire ({b_wire}), got {b_delivered}"
);
anyhow::ensure!(
a_delivered < U256::from(whole_wire) && b_delivered < U256::from(whole_wire),
"neither holder may be paid for the whole blob: A={a_delivered}, B={b_delivered}, \
whole={whole_wire}"
);
let paid_providers: std::collections::HashSet<Address> = progress_log(&recorded)?
.into_iter()
.map(|(p, ..)| p)
.collect();
anyhow::ensure!(
paid_providers == std::collections::HashSet::from([a_eth, b_eth]),
"both holders' lanes must have persisted a watermark, got {paid_providers:?}"
);
anyhow::ensure!(
counter_value(&s_metrics, "node_pull_leg_no_progress_total")? == 0,
"no clean leg may stall a healthy two-holder assembly"
);
shutdown([task_s, task_a, task_b], [&leaf_ep, &ep_s, &ep_a, &ep_b]).await?;
Ok(())
}
async fn spawn_dht_record_server(
hash: Hash,
holder: DhtNodeId,
coverage: Coverage,
) -> Result<(
iroh::PublicKey,
std::net::SocketAddr,
iroh::Endpoint,
tokio::task::JoinHandle<()>,
)> {
use decdn_node::dht::rate_limit::DhtRateLimitConfig;
use decdn_node::dht::{DhtRateLimiter, InsertOutcome, RecordStore, RecordStoreConfig};
use iroh::protocol::ProtocolHandler;
let sk = fresh_key();
let id = sk.public();
let metrics = Arc::new(Metrics::new());
let rate_limiter = Arc::new(DhtRateLimiter::new(
&DhtRateLimitConfig {
per_peer_rate_per_sec: 1e6,
per_peer_burst: u32::MAX,
per_ip_rate_per_sec: 1e6,
per_ip_burst: u32::MAX,
global_rate_per_sec: 1e6,
global_burst: u32::MAX,
max_tracked_per_ip: 4096,
max_tracked_per_peer: 4096,
},
Arc::clone(&metrics),
));
let records = Arc::new(Mutex::new(RecordStore::new(RecordStoreConfig::default())));
let receive_us = u64::try_from(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_micros()),
)
.unwrap_or(u64::MAX);
let outcome = records
.lock()
.map_err(|_| anyhow::anyhow!("record store mutex poisoned"))?
.insert_at(
holder,
ContentHash::from_bytes(*hash.as_bytes()),
coverage,
receive_us,
);
anyhow::ensure!(
matches!(outcome, InsertOutcome::Inserted),
"fixture record insert must succeed: {outcome:?}"
);
let handler = Arc::new(decdn_node::handlers::dht::DhtHandler::with_routing(
id,
Arc::new(Mutex::new(RoutingTable::new(DhtNodeId::from_bytes(
*id.as_bytes(),
)))),
rate_limiter,
permissive_limiter(&metrics),
Arc::clone(&metrics),
Arc::new(ConfigStakerSet::new(HashSet::new())) as Arc<dyn StakerSet>,
records,
));
let (ep, addr) = local_endpoint(sk, vec![decdn_protocol::ALPN_DHT.to_vec()]).await?;
let task = {
let ep = ep.clone();
tokio::spawn(async move {
while let Some(incoming) = ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let handler = Arc::clone(&handler);
tokio::spawn(async move {
let _ = handler.accept(conn).await;
});
}
})
};
Ok((id, addr, ep, task))
}
fn two_block_blob() -> Result<(decdn_protocol::TestBlockSizeGuard, Vec<u8>, Hash)> {
let block: u64 = 16 * 1024;
let guard = decdn_protocol::override_discovery_block_bytes_for_test(block);
anyhow::ensure!(
decdn_protocol::discovery_block_bytes() == block,
"override did not take"
);
let payload_len = usize::try_from(2 * block).unwrap_or(usize::MAX);
let payload: Vec<u8> = (0..payload_len)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect();
let hash = Hash::new(&payload);
anyhow::ensure!(
decdn_protocol::num_blocks(2 * block) == 2,
"the blob must span exactly two discovery blocks under the override"
);
Ok((guard, payload, hash))
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum DiscoveryCase {
DhtPartial,
CachedPartial,
CachedSpanning,
DhtSpanning,
DirectoryPastFanout,
NoCoveringCandidate,
}
impl DiscoveryCase {
const fn probe_rounds(self) -> u64 {
match self {
Self::CachedSpanning => 0,
Self::CachedPartial | Self::DhtSpanning | Self::NoCoveringCandidate => 1,
Self::DhtPartial | Self::DirectoryPastFanout => 2,
}
}
}
#[allow(clippy::too_many_lines)]
async fn serve_miss_discovery_case(case: DiscoveryCase, byte_len: u64) -> Result<()> {
let spans = support::capture_spans();
let (_block_guard, payload, hash) = two_block_blob()?;
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let partial = Coverage::from_block_indices(2, [0].into_iter());
let ab_pool_id = B256::repeat_byte(0xA3);
let s_buyer = Arc::new(PrivateKeySigner::random());
let (p_id, p_addr, p_eth, ep_p, task_p, _store_p) =
spawn_partial_holder(&payload, ab_pool_id, s_buyer.address(), partial.clone(), 0).await?;
let (o_id, o_addr, o_eth, ep_o, task_o, store_o) = spawn_partial_holder(
&payload,
ab_pool_id,
s_buyer.address(),
Coverage::full(2),
0,
)
.await?;
let p_dht = DhtNodeId::from_bytes(*p_id.as_bytes());
let o_dht = DhtNodeId::from_bytes(*o_id.as_bytes());
let d_dht = DhtNodeId::from_bytes(*fresh_key().public().as_bytes());
let probed = |node_id: DhtNodeId, coverage: Coverage| ProbedProvider {
node_id,
rate_per_mb: RATE,
rtt_ms: 1,
coverage,
total_bytes_hint: Some(total_bytes),
};
let directory = match case {
DiscoveryCase::DhtPartial
| DiscoveryCase::CachedPartial
| DiscoveryCase::CachedSpanning => {
vec![o_dht]
}
DiscoveryCase::DhtSpanning => vec![p_dht],
DiscoveryCase::DirectoryPastFanout => vec![p_dht, o_dht],
DiscoveryCase::NoCoveringCandidate => vec![p_dht, d_dht],
};
let mut discovery = FixtureDiscovery {
stakers: HashSet::from([p_dht, o_dht, d_dht]),
directory: HashMap::from([(U256::ZERO, directory)]),
dht_peers: Vec::new(),
probe_fanout: if case == DiscoveryCase::DirectoryPastFanout {
1
} else {
5
},
};
let probe_cache = PositiveProbeCache::new();
let target = ContentHash::from_bytes(*hash.as_bytes());
let dht_record = match case {
DiscoveryCase::DhtPartial => Some((p_dht, partial.clone())),
DiscoveryCase::DhtSpanning => Some((o_dht, Coverage::full(2))),
DiscoveryCase::CachedPartial => {
probe_cache.insert(target, vec![probed(p_dht, partial.clone())]);
None
}
DiscoveryCase::CachedSpanning => {
probe_cache.insert(
target,
vec![
probed(p_dht, partial.clone()),
probed(o_dht, Coverage::full(2)),
],
);
None
}
DiscoveryCase::DirectoryPastFanout | DiscoveryCase::NoCoveringCandidate => None,
};
let mut dht_server = None;
if let Some((holder, coverage)) = dht_record {
let (dht_id, dht_addr, dht_ep, dht_task) =
spawn_dht_record_server(hash, holder, coverage).await?;
discovery.dht_peers.push((dht_id, dht_addr));
dht_server = Some((dht_ep, dht_task));
}
let addr_map = HashMap::from([(p_dht, p_eth), (o_dht, o_eth)]);
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x2F);
let (handler_s, s_target, ep_s, _recorded, cache_s, s_operator, s_metrics, _) =
build_serving_node(
hash,
ab_pool_id,
&s_buyer,
discovery,
probe_cache,
addr_map,
&[(p_id, p_addr), (o_id, o_addr)],
&[(leaf_pool_id, leaf_eth.address())],
U256::from(DEPOSIT_MICRO_USDC),
)
.await?;
let task_s = spawn_server(ep_s.clone(), handler_s);
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
let pulled = if byte_len == 0 {
leaf_paced_pull(
&leaf_ep,
s_target,
leaf_node_id,
&leaf_eth,
s_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await
} else {
leaf_paced_pull_ranged(
&leaf_ep,
s_target,
leaf_node_id,
&leaf_eth,
s_operator,
leaf_pool_id,
hash,
RATE,
byte_len,
)
.await
};
if case == DiscoveryCase::NoCoveringCandidate {
let Err(err) = pulled else {
anyhow::bail!("S must refuse when no candidate covers block 1");
};
anyhow::ensure!(
format!("{err:#}").contains("delivery refused"),
"S must refuse before committing, not truncate after `ok: true`: {err:#}"
);
let lines: Vec<_> = spans
.events(
"serve-miss: refusing the request",
"hash",
&hash.to_string(),
)
.into_iter()
.filter(|l| l.get("path").map(String::as_str) == Some("coverage_gate"))
.collect();
anyhow::ensure!(
!lines.is_empty(),
"no coverage_gate refusal line for the hash"
);
for line in &lines {
anyhow::ensure!(
line.get("reason").map(String::as_str) == Some("cache_miss")
&& line.contains_key("candidates")
&& line.contains_key("first_uncovered_chunk"),
"coverage_gate line: {line:?}"
);
}
} else {
let outcome = pulled?;
anyhow::ensure!(outcome.completed, "leaf delivery did not complete");
anyhow::ensure!(outcome.hash_ok, "leaf received bytes failed the hash check");
anyhow::ensure!(
outcome.received == total_bytes,
"leaf received {} of {total_bytes} bytes",
outcome.received
);
anyhow::ensure!(
cache_s.has(hash).await?,
"S must promote the assembled blob on a complete delivery"
);
let o_delivered = store_o
.get(LaneKey {
pool_id: ab_pool_id,
signer: s_buyer.address(),
provider: o_eth,
})?
.ok_or_else(|| anyhow::anyhow!("origin lane vanished"))?
.last_bytes_delivered();
anyhow::ensure!(
o_delivered > U256::ZERO,
"the origin must have served the block the partial holder lacks"
);
}
let rounds = counter_value(&s_metrics, "probe_collection_latency_seconds_count")?;
anyhow::ensure!(
rounds == case.probe_rounds(),
"expected {} probe rounds, got {rounds}",
case.probe_rounds()
);
let expected_cache = match case {
DiscoveryCase::CachedPartial => Some((0, 1)),
DiscoveryCase::CachedSpanning => Some((1, 0)),
_ => None,
};
if let Some((hits, misses)) = expected_cache {
let got = (
counter_value(&s_metrics, "probe_cache_hits_total")?,
counter_value(&s_metrics, "probe_cache_misses_total")?,
);
anyhow::ensure!(
got == (hits, misses),
"expected (hits, misses) = ({hits}, {misses}), got {got:?}"
);
}
match dht_server {
Some((dht_ep, dht_task)) => {
shutdown(
[task_s, task_p, task_o, dht_task],
[&leaf_ep, &ep_s, &ep_p, &ep_o, &dht_ep],
)
.await?;
}
None => {
shutdown([task_s, task_p, task_o], [&leaf_ep, &ep_s, &ep_p, &ep_o]).await?;
}
}
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_dht_answer_of_partial_holders_only_pulls_in_the_directory_origin() -> Result<()> {
serve_miss_discovery_case(DiscoveryCase::DhtPartial, 0).await
}
#[tokio::test(flavor = "multi_thread")]
async fn a_partial_only_probe_cache_hit_falls_through_to_the_directory_origin() -> Result<()> {
serve_miss_discovery_case(DiscoveryCase::CachedPartial, 0).await
}
#[tokio::test(flavor = "multi_thread")]
async fn a_spanning_probe_cache_hit_serves_without_probing() -> Result<()> {
serve_miss_discovery_case(DiscoveryCase::CachedSpanning, 0).await
}
#[tokio::test(flavor = "multi_thread")]
async fn spanning_dht_holders_skip_the_origin_supplement() -> Result<()> {
serve_miss_discovery_case(DiscoveryCase::DhtSpanning, 0).await
}
#[tokio::test(flavor = "multi_thread")]
async fn a_directory_origin_past_the_probe_fanout_is_probed() -> Result<()> {
serve_miss_discovery_case(DiscoveryCase::DirectoryPastFanout, 0).await
}
#[tokio::test(flavor = "multi_thread")]
async fn no_covering_candidate_refuses_before_commit() -> Result<()> {
serve_miss_discovery_case(DiscoveryCase::NoCoveringCandidate, 0).await
}
#[tokio::test(flavor = "multi_thread")]
async fn no_covering_candidate_with_end_past_the_blob_refuses_before_commit() -> Result<()> {
let total_bytes: u64 = 2 * 16 * 1024;
serve_miss_discovery_case(DiscoveryCase::NoCoveringCandidate, total_bytes + 4096).await
}
struct HolderPull {
first_blocks: &'static [u32],
leaf_is_first: bool,
full_holder_refusals: usize,
}
struct FirstHolderSeen {
refused_misses: u64,
paid_bytes: U256,
inbound_streams: u64,
full_holder_streams: u64,
}
fn inbound_streams_ended(metrics: &Arc<Metrics>) -> Result<u64> {
let text = metrics
.encode()
.map_err(|e| anyhow::anyhow!("encode metrics: {e}"))?;
let count = |name: &str| {
let prefix = format!("decdn_{name}{{direction=\"inbound\"}} ");
text.lines()
.find_map(|l| l.strip_prefix(&prefix))
.and_then(|v| v.trim().parse::<u64>().ok())
.unwrap_or(0)
};
Ok(count("streams_completed_total") + count("streams_failed_total"))
}
#[allow(clippy::too_many_lines)]
async fn holder_pull_case(case: HolderPull) -> Result<FirstHolderSeen> {
let (_block_guard, payload, hash) = two_block_blob()?;
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA4);
let s_buyer = Arc::new(PrivateKeySigner::random());
let (f_id, f_addr, f_eth, ep_f, task_f, store_f, metrics_f) =
spawn_block_holder(&payload, ab_pool_id, s_buyer.address(), case.first_blocks).await?;
let (cache_o, hash_o, tmp_o) = cache_with_blob(&payload).await?;
anyhow::ensure!(hash_o == hash, "full holder fixture hash mismatch");
std::mem::forget(tmp_o);
let (o_id, o_addr, o_eth, ep_o, task_o, store_o, metrics_o) = spawn_holder_on(
cache_o,
&payload,
ab_pool_id,
s_buyer.address(),
Coverage::full(2),
case.full_holder_refusals,
)
.await?;
let f_dht = DhtNodeId::from_bytes(*f_id.as_bytes());
let o_dht = DhtNodeId::from_bytes(*o_id.as_bytes());
let first_coverage = Coverage::from_block_indices(2, case.first_blocks.iter().copied());
let probed = |node_id: DhtNodeId, coverage: Coverage, rtt_ms: u32| ProbedProvider {
node_id,
rate_per_mb: RATE,
rtt_ms,
coverage,
total_bytes_hint: Some(total_bytes),
};
let probe_cache = PositiveProbeCache::new();
probe_cache.insert(
ContentHash::from_bytes(*hash.as_bytes()),
vec![
probed(f_dht, first_coverage, 1),
probed(o_dht, Coverage::full(2), 500),
],
);
let discovery = FixtureDiscovery {
stakers: HashSet::from([f_dht, o_dht]),
directory: HashMap::from([(U256::ZERO, vec![o_dht])]),
dht_peers: Vec::new(),
probe_fanout: 5,
};
let addr_map = HashMap::from([(f_dht, f_eth), (o_dht, o_eth)]);
let leaf_eth = Arc::new(PrivateKeySigner::random());
let leaf_pool_id = B256::repeat_byte(0x2E);
let (handler_s, s_target, ep_s, _recorded, _cache_s, s_operator, _s_metrics, _) =
build_serving_node(
hash,
ab_pool_id,
&s_buyer,
discovery,
probe_cache,
addr_map,
&[(f_id, f_addr), (o_id, o_addr)],
&[(leaf_pool_id, leaf_eth.address())],
U256::from(DEPOSIT_MICRO_USDC),
)
.await?;
let task_s = spawn_server(ep_s.clone(), handler_s);
let (leaf_ep, leaf_node_id, own_leaf) = if case.leaf_is_first {
(ep_f.clone(), B256::from(*f_id.as_bytes()), false)
} else {
let leaf_sk = fresh_key();
let id = B256::from(*leaf_sk.public().as_bytes());
let (ep, _) = local_endpoint(leaf_sk, vec![]).await?;
(ep, id, true)
};
let outcome = leaf_paced_pull(
&leaf_ep,
s_target,
leaf_node_id,
&leaf_eth,
s_operator,
leaf_pool_id,
hash,
RATE,
None,
)
.await?;
anyhow::ensure!(outcome.completed, "leaf delivery did not complete");
anyhow::ensure!(outcome.hash_ok, "leaf received bytes failed the hash check");
anyhow::ensure!(
outcome.received == total_bytes,
"leaf received {} of {total_bytes} bytes",
outcome.received
);
let paid = |store: &Arc<MemoryPoolStateStore>, provider: Address| -> Result<U256> {
Ok(store
.get(LaneKey {
pool_id: ab_pool_id,
signer: s_buyer.address(),
provider,
})?
.map_or(U256::ZERO, |lane| lane.last_bytes_delivered()))
};
anyhow::ensure!(
paid(&store_o, o_eth)? > U256::ZERO,
"the full holder must have served S"
);
let seen = FirstHolderSeen {
refused_misses: counter_value(&metrics_f, "serve_stream_rejected_cache_miss_total")?,
paid_bytes: paid(&store_f, f_eth)?,
inbound_streams: inbound_streams_ended(&metrics_f)?,
full_holder_streams: inbound_streams_ended(&metrics_o)?,
};
if own_leaf {
shutdown([task_s, task_f, task_o], [&leaf_ep, &ep_s, &ep_f, &ep_o]).await?;
} else {
shutdown([task_s, task_f, task_o], [&ep_s, &ep_f, &ep_o]).await?;
}
Ok(seen)
}
#[tokio::test(flavor = "multi_thread")]
async fn a_serve_miss_asks_a_partial_holder_only_for_blocks_it_holds() -> Result<()> {
let seen = holder_pull_case(HolderPull {
first_blocks: &[1],
leaf_is_first: false,
full_holder_refusals: 0,
})
.await?;
anyhow::ensure!(
seen.refused_misses == 0,
"the partial holder was asked for bytes it does not hold ({} refused misses)",
seen.refused_misses
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_refused_first_run_handshake_asks_the_partial_holder_only_for_blocks_it_holds()
-> Result<()> {
let seen = holder_pull_case(HolderPull {
first_blocks: &[1],
leaf_is_first: false,
full_holder_refusals: 2,
})
.await?;
anyhow::ensure!(
seen.inbound_streams > 0,
"the handshake never reached the partial holder after the full holder refused"
);
anyhow::ensure!(
seen.refused_misses == 0,
"the partial holder was asked for bytes it does not hold ({} refused misses)",
seen.refused_misses
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_serve_miss_handshakes_with_the_first_runs_holder() -> Result<()> {
let seen = holder_pull_case(HolderPull {
first_blocks: &[1],
leaf_is_first: false,
full_holder_refusals: 0,
})
.await?;
anyhow::ensure!(
seen.inbound_streams == 0,
"S opened {} streams to the first-ranked holder, which the first run does not use",
seen.inbound_streams
);
anyhow::ensure!(
seen.full_holder_streams == 1,
"the full holder ended {} streams; the first run must adopt the handshake's pull",
seen.full_holder_streams
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_serve_miss_never_pulls_from_its_requester() -> Result<()> {
let seen = holder_pull_case(HolderPull {
first_blocks: &[0, 1],
leaf_is_first: true,
full_holder_refusals: 0,
})
.await?;
anyhow::ensure!(
seen.paid_bytes == U256::ZERO,
"S pulled {} bytes from the node that asked it",
seen.paid_bytes
);
anyhow::ensure!(
seen.refused_misses == 0,
"the requester was sent a request it had to refuse"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn a_whole_blob_request_from_a_node_is_refused_while_its_upstream_open_runs() -> Result<()> {
let (_block_guard, payload, hash) = two_block_blob()?;
let total_bytes = u64::try_from(payload.len()).unwrap_or(u64::MAX);
let ab_pool_id = B256::repeat_byte(0xA5);
let s_buyer = Arc::new(PrivateKeySigner::random());
let u_sk = fresh_key();
let u_id = u_sk.public();
let u_eth = Arc::new(PrivateKeySigner::random());
let (ep_u, addr_u) =
local_endpoint(u_sk, vec![ALPN_PROBE.to_vec(), ALPN_CLIENT.to_vec()]).await?;
let u_streams = Arc::new(AtomicUsize::new(0));
let task_u = spawn_a_counting_stalling_server(
ep_u.clone(),
Arc::clone(&u_eth),
slash_domain(),
total_bytes,
RATE,
Arc::clone(&u_streams),
);
let u_dht = DhtNodeId::from_bytes(*u_id.as_bytes());
let leaves: Vec<(iroh::SecretKey, Arc<PrivateKeySigner>, B256)> = (0u8..4)
.map(|i| {
(
fresh_key(),
Arc::new(PrivateKeySigner::random()),
B256::repeat_byte(0x60 + i),
)
})
.collect();
let [
(sk1, eth1, ch1),
(sk2, eth2, ch2),
(sk3, eth3, ch3),
(sk4, eth4, ch4),
] = <[_; 4]>::try_from(leaves).map_err(|_| anyhow::anyhow!("four leaves"))?;
let mut discovery = FixtureDiscovery::directory_only(vec![u_dht], U256::ZERO);
for sk in [&sk1, &sk2, &sk4] {
discovery
.stakers
.insert(DhtNodeId::from_bytes(*sk.public().as_bytes()));
}
let (handler_s, s_target, ep_s, _recorded, _cache_s, s_operator, s_metrics, _) =
build_serving_node(
hash,
ab_pool_id,
&s_buyer,
discovery,
PositiveProbeCache::new(),
HashMap::from([(u_dht, u_eth.address())]),
&[(u_id, addr_u)],
&[
(ch1, eth1.address()),
(ch2, eth2.address()),
(ch3, eth3.address()),
(ch4, eth4.address()),
],
U256::from(DEPOSIT_MICRO_USDC),
)
.await?;
let task_s = spawn_server_concurrent(ep_s.clone(), handler_s);
let (ep1, _) = local_endpoint(sk1.clone(), vec![]).await?;
let (ep2, _) = local_endpoint(sk2.clone(), vec![]).await?;
let (ep3, _) = local_endpoint(sk3.clone(), vec![]).await?;
let (ep4, _) = local_endpoint(sk4.clone(), vec![]).await?;
let wait_for_opens = |n: usize, what: &'static str| {
let u_streams = Arc::clone(&u_streams);
async move {
tokio::time::timeout(Duration::from_secs(20), async {
while u_streams.load(Ordering::SeqCst) < n {
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.map_err(|_| anyhow::anyhow!("{what}"))
}
};
let first = {
let (ep, target, eth) = (ep1.clone(), s_target.clone(), Arc::clone(ð1));
let id = B256::from(*sk1.public().as_bytes());
tokio::spawn(async move {
leaf_paced_pull(&ep, target, id, ð, s_operator, ch1, hash, RATE, None).await
})
};
wait_for_opens(1, "a staker with no open in progress was refused").await?;
let refused = tokio::time::timeout(
Duration::from_secs(5),
leaf_paced_pull(
&ep2,
s_target.clone(),
B256::from(*sk2.public().as_bytes()),
ð2,
s_operator,
ch2,
hash,
RATE,
None,
),
)
.await
.map_err(|_| anyhow::anyhow!("the node's whole-blob request was not answered at once"))?;
let err = refused
.err()
.ok_or_else(|| anyhow::anyhow!("the node's whole-blob request was served"))?;
anyhow::ensure!(
format!("{err:#}").contains("NotFound"),
"the refusal is a miss on the wire: {err:#}"
);
tokio::time::sleep(Duration::from_millis(250)).await;
anyhow::ensure!(
u_streams.load(Ordering::SeqCst) == 1,
"the refused request opened a second upstream leg"
);
anyhow::ensure!(
counter_value(&s_metrics, "serve_stream_rejected_pull_loop_guard_total")? == 1,
"the refusal counts on its own reason"
);
anyhow::ensure!(
counter_value(&s_metrics, "serve_stream_rejected_cache_miss_total")? == 0,
"the refusal is not counted as a cache miss"
);
let third = {
let (ep, target, eth) = (ep3.clone(), s_target.clone(), Arc::clone(ð3));
let id = B256::from(*sk3.public().as_bytes());
tokio::spawn(async move {
leaf_paced_pull(&ep, target, id, ð, s_operator, ch3, hash, RATE, None).await
})
};
wait_for_opens(2, "the client's whole-blob request was refused").await?;
let fourth = {
let (ep, target, eth) = (ep4.clone(), s_target.clone(), Arc::clone(ð4));
let id = B256::from(*sk4.public().as_bytes());
let len = total_bytes / 2;
tokio::spawn(async move {
leaf_paced_pull_ranged(&ep, target, id, ð, s_operator, ch4, hash, RATE, len).await
})
};
wait_for_opens(3, "the staker's ranged request was refused").await?;
anyhow::ensure!(
counter_value(&s_metrics, "serve_stream_rejected_pull_loop_guard_total")? == 1,
"only the one whole-blob request from a staker mid-open was refused"
);
first.abort();
third.abort();
fourth.abort();
shutdown([task_s, task_u], [&ep1, &ep2, &ep3, &ep4, &ep_s, &ep_u]).await?;
Ok(())
}
#[allow(clippy::too_many_lines)]
async fn sole_coverer_refusal_is_retried(leaves: usize) -> Result<()> {
let block: u64 = 16 * 1024;
let _block_guard = decdn_protocol::override_discovery_block_bytes_for_test(block);
anyhow::ensure!(
decdn_protocol::discovery_block_bytes() == block,
"override did not take"
);
let payload_len = usize::try_from(2 * block).unwrap_or(usize::MAX);
let payload: Vec<u8> = (0..payload_len)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect();
let hash = Hash::new(&payload);
let total_bytes = 2 * block;
let ab_pool_id = B256::repeat_byte(0xA2);
let s_buyer = Arc::new(PrivateKeySigner::random());
let (a_id, a_addr, a_eth, ep_a, task_a, _store_a) = spawn_partial_holder(
&payload,
ab_pool_id,
s_buyer.address(),
Coverage::from_block_indices(2, [0].into_iter()),
0,
)
.await?;
let (b_id, b_addr, b_eth, ep_b, task_b, store_b) = spawn_partial_holder(
&payload,
ab_pool_id,
s_buyer.address(),
Coverage::from_block_indices(2, [1].into_iter()),
2,
)
.await?;
let (providers, addr_map) = two_providers(
DhtNodeId::from_bytes(*a_id.as_bytes()),
a_eth,
DhtNodeId::from_bytes(*b_id.as_bytes()),
b_eth,
);
let leaf_eths: Vec<Arc<PrivateKeySigner>> = (0..leaves)
.map(|_| Arc::new(PrivateKeySigner::random()))
.collect();
let leaf_lanes: Vec<(B256, Address)> = leaf_eths
.iter()
.zip(0x2Fu8..)
.map(|(eth, byte)| (B256::repeat_byte(byte), eth.address()))
.collect();
let (handler_s, s_target, ep_s, _recorded, cache_s, s_operator, s_metrics, origin_s) =
build_serving_node(
hash,
ab_pool_id,
&s_buyer,
FixtureDiscovery::directory_only(providers, U256::ZERO),
PositiveProbeCache::new(),
addr_map,
&[(a_id, a_addr), (b_id, b_addr)],
&leaf_lanes,
U256::from(DEPOSIT_MICRO_USDC),
)
.await?;
let task_s = spawn_server(ep_s.clone(), handler_s);
let mut leaf_eps = Vec::new();
let mut pulls = Vec::new();
for (eth, &(pool_id, _)) in leaf_eths.iter().zip(&leaf_lanes) {
let leaf_sk = fresh_key();
let leaf_node_id = B256::from(*leaf_sk.public().as_bytes());
let (leaf_ep, _) = local_endpoint(leaf_sk, vec![]).await?;
leaf_eps.push(leaf_ep.clone());
let eth = Arc::clone(eth);
let target = s_target.clone();
pulls.push(tokio::spawn(async move {
leaf_paced_pull(
&leaf_ep,
target,
leaf_node_id,
ð,
s_operator,
pool_id,
hash,
RATE,
None,
)
.await
}));
}
for (ix, pull) in pulls.into_iter().enumerate() {
let outcome = pull.await??;
anyhow::ensure!(
outcome.completed,
"leaf {ix}'s paid delivery must not end short on a refused leg"
);
anyhow::ensure!(outcome.hash_ok, "leaf {ix}'s bytes failed the hash check");
anyhow::ensure!(
outcome.received == total_bytes,
"leaf {ix} received {} of {total_bytes} bytes",
outcome.received
);
}
anyhow::ensure!(
cache_s.has(hash).await?,
"S must promote the assembled blob on a complete delivery"
);
anyhow::ensure!(
counter_value(&s_metrics, "node_pull_backpressure_backoffs_total")? >= 1,
"the refused run must have been waited on, not dropped"
);
anyhow::ensure!(
counter_value(&s_metrics, "node_pull_backpressure_exhausted_total")? == 0,
"the refusal cleared inside the wait budget"
);
anyhow::ensure!(
counter_value(&s_metrics, "node_pull_leg_no_progress_total")? == 0,
"every leg of the retried assembly made progress (#2194)"
);
anyhow::ensure!(
counter_value(&s_metrics, "node_pull_refused_total")? >= 1
&& counter_value(&s_metrics, "node_pull_refused_unattributable_total")? >= 1,
"the refusal is still metered"
);
anyhow::ensure!(
!origin_s.is_suppressed(b_id, hash),
"a backpressure refusal must not suppress the holder for the hash"
);
let lane_b = LaneKey {
pool_id: ab_pool_id,
signer: s_buyer.address(),
provider: b_eth,
};
let b_delivered = store_b
.get(lane_b)?
.ok_or_else(|| anyhow::anyhow!("holder B lane vanished"))?
.last_bytes_delivered();
anyhow::ensure!(
b_delivered > U256::ZERO,
"B must have served block 1 on the retry"
);
shutdown([task_s, task_a, task_b], [&ep_s, &ep_a, &ep_b]).await?;
for leaf_ep in leaf_eps {
leaf_ep.close().await;
}
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn a_sole_coverer_backpressure_refusal_is_retried_not_dropped() -> Result<()> {
sole_coverer_refusal_is_retried(1).await
}
#[tokio::test(flavor = "multi_thread")]
async fn sibling_streams_survive_a_sole_coverer_backpressure_refusal() -> Result<()> {
sole_coverer_refusal_is_retried(2).await
}