use std::sync::Arc;
use std::time::Duration;
use alloy::dyn_abi::Eip712Domain;
use alloy::primitives::{Address, B256, U256};
use alloy::signers::SignerSync;
use alloy::signers::local::PrivateKeySigner;
use bao_tree::io::outboard::PreOrderMemOutboard;
use bytes::BytesMut;
use decdn_cache::range_pull::{IROH_BLOCK_SIZE, align_range, encode_verified_range};
use decdn_cache::{CacheEngine, Hash, HttpOrigin, Origin};
use decdn_incentive::{
EPHEMERAL_BINDING_NONCE, LaneState, MemoryPoolStateStore, PoolStateStore, Voucher,
bind_node_id_domain, binding_signing_hash, signed_to_wire_voucher, slash_judge_domain,
voucher_domain,
};
use decdn_node::handlers::client::ClientHandler;
use decdn_node::metrics::Metrics;
use decdn_protocol::client::{ClientBinding, ClientMessage, StreamRequest, StreamRequestExt};
use decdn_protocol::{ALPN_CLIENT, CHUNK_BYTES, MB_BYTES, encode_stream_request, write_frame};
use iroh::EndpointAddr;
use iroh::endpoint::SendStream;
use wiremock::matchers::{header, header_exists, method, path};
use wiremock::{Mock, MockServer, Request, ResponseTemplate};
mod support;
use support::{
HandlerDomains, build_handler_full_configured, fresh_key, local_endpoint, permissive_limiter,
read_client_msg, read_stream_response, shutdown, spawn_server, write_client_msg,
};
const CHAIN_ID: u64 = 421_614;
const RATE_PER_MB: u64 = 10;
fn slash_dom() -> Eip712Domain {
slash_judge_domain(CHAIN_ID, Address::repeat_byte(0x11))
}
fn voucher_dom() -> Eip712Domain {
voucher_domain(CHAIN_ID, Address::repeat_byte(0x34))
}
fn binding_dom() -> Eip712Domain {
bind_node_id_domain(CHAIN_ID, Address::repeat_byte(0x99))
}
fn domains() -> HandlerDomains {
HandlerDomains {
slash: slash_dom(),
voucher: voucher_dom(),
binding: binding_dom(),
}
}
fn blob_with_outboard() -> (Vec<u8>, Vec<u8>, Hash) {
let blob: Vec<u8> = (0..200 * 1024u32)
.map(|i| u8::try_from(i % 251).unwrap_or(0))
.collect();
let ob = PreOrderMemOutboard::create(&blob, IROH_BLOCK_SIZE);
let hash = Hash::from_bytes(*ob.root.as_bytes());
(blob, ob.data, hash)
}
async fn handler_over_http_origin(
origin_uri: &str,
pool_id: B256,
client: Address,
server_eth: &Arc<PrivateKeySigner>,
server_id: iroh::PublicKey,
pull_through: Option<Duration>,
) -> anyhow::Result<(
Arc<ClientHandler>,
CacheEngine,
Arc<Metrics>,
tempfile::TempDir,
)> {
let store = Arc::new(MemoryPoolStateStore::new());
store.record(&LaneState::hydrate(
pool_id,
client, server_eth.address(), U256::from(10_000_000u64),
0, U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let cache_dir = tempfile::tempdir()?;
let origin = Arc::new(HttpOrigin::parse(origin_uri)?);
let cache = CacheEngine::open(cache_dir.path(), vec![origin as Arc<dyn Origin>], 16).await?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let store_dyn: Arc<dyn PoolStateStore> = store;
let handler = build_handler_full_configured(
server_id,
server_eth,
&metrics,
limiter,
cache.clone(),
store_dyn,
RATE_PER_MB,
&domains(),
16,
|deps| deps.pull_through = pull_through,
)?;
Ok((handler, cache, metrics, cache_dir))
}
fn counter_value(metrics: &Arc<Metrics>, name: &str) -> anyhow::Result<u64> {
let text = metrics
.encode()
.map_err(|e| anyhow::anyhow!("encode metrics: {e}"))?;
let prefix = format!("decdn_{name} ");
Ok(text
.lines()
.find_map(|l| l.strip_prefix(&prefix))
.and_then(|v| v.trim().parse::<u64>().ok())
.unwrap_or(0))
}
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
async fn ranged_paid_pull(
client_ep: &iroh::Endpoint,
target: EndpointAddr,
client_node_id: B256,
client_eth: &Arc<PrivateKeySigner>,
pool_id: B256,
provider: Address,
hash: Hash,
byte_offset: u64,
byte_len: u64,
rate: u64,
) -> anyhow::Result<Vec<u8>> {
let conn = client_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("open_bi: {e}"))?;
let binding_hash =
binding_signing_hash(client_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = client_eth
.sign_hash_sync(&binding_hash)?
.as_bytes()
.to_vec();
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: client_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset,
byte_len,
timestamp_us: 0x9001,
};
let payload =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame_to(&mut send, &payload).await?;
let (resp, resp_ext) = read_stream_response(&mut recv).await?;
anyhow::ensure!(resp.body.ok, "delivery refused: {:?}", resp_ext.error);
if byte_len == 0 {
resp.body
.total_bytes
.checked_sub(byte_offset)
.ok_or_else(|| anyhow::anyhow!("response total_bytes is before byte_offset"))?;
} else {
anyhow::ensure!(
byte_offset < resp.body.total_bytes,
"response total_bytes is at or before the requested offset"
);
}
let aligned =
decdn_cache::range_pull::align_range_clamped(byte_offset, byte_len, resp.body.total_bytes)
.map_err(|e| anyhow::anyhow!("align range: {e}"))?;
let expected_wire =
decdn_cache::range_pull::bao_encoded_size(resp.body.total_bytes, aligned.chunk_ranges());
let interval_bytes = CHUNK_BYTES;
let mut buf = BytesMut::new();
let mut cumulative: u64 = 0;
let mut unvouchered: u64 = 0;
loop {
match read_client_msg(&mut recv).await? {
ClientMessage::ChunkData(chunk) => {
buf.extend_from_slice(chunk.bytes());
let len = u64::try_from(chunk.bytes().len()).unwrap_or(u64::MAX);
cumulative = cumulative.saturating_add(len);
unvouchered = unvouchered.saturating_add(len);
let boundary = interval_bytes > 0 && unvouchered >= interval_bytes;
let closing = cumulative >= expected_wire && unvouchered > 0;
if boundary || closing {
let amount = U256::from(cumulative)
.saturating_mul(U256::from(rate))
.div_ceil(U256::from(MB_BYTES));
let signed = Voucher {
pool_id,
signer: client_eth.address(),
provider,
amount,
bytes_delivered: U256::from(cumulative),
chain_root: B256::ZERO,
chunk_price: U256::ZERO,
}
.sign(client_eth.as_ref(), &voucher_dom())
.map_err(|e| anyhow::anyhow!("sign voucher: {e}"))?;
write_client_msg(
&mut send,
&ClientMessage::Voucher(signed_to_wire_voucher(&signed)?),
)
.await?;
unvouchered = 0;
}
}
ClientMessage::StreamEnd => break,
ClientMessage::StreamError(e) => anyhow::bail!("stream error mid-delivery: {e:?}"),
other => anyhow::bail!("unexpected message mid-delivery: {other:?}"),
}
}
conn.close(0u32.into(), b"done");
decode_bao_range(hash, resp.body.total_bytes, byte_offset, byte_len, &buf)
}
fn decode_bao_range(
hash: Hash,
total_bytes: u64,
byte_offset: u64,
byte_len: u64,
wire: &[u8],
) -> anyhow::Result<Vec<u8>> {
use bao_tree::BaoTree;
use bao_tree::io::BaoContentItem;
use bao_tree::io::sync::DecodeResponseIter;
let aligned = decdn_cache::range_pull::align_range_clamped(byte_offset, byte_len, total_bytes)
.map_err(|e| anyhow::anyhow!("align range: {e}"))?;
let tree = BaoTree::new(total_bytes, decdn_cache::range_pull::IROH_BLOCK_SIZE);
let reader = std::io::Cursor::new(wire);
let mut plaintext = Vec::new();
for item in DecodeResponseIter::new(hash.into(), tree, reader, aligned.chunk_ranges().as_ref())
{
match item.map_err(|e| anyhow::anyhow!("bao decode: {e}"))? {
BaoContentItem::Leaf(leaf) => plaintext.extend_from_slice(&leaf.data),
BaoContentItem::Parent(_) => {}
}
}
let lead = usize::try_from(byte_offset.saturating_sub(aligned.fetch_start()))?;
let want = if byte_len == 0 {
plaintext.len().saturating_sub(lead)
} else {
let end = byte_offset.saturating_add(byte_len).min(total_bytes);
usize::try_from(end.saturating_sub(byte_offset))?
};
let end = lead.saturating_add(want);
let slice = plaintext
.get(lead..end)
.ok_or_else(|| anyhow::anyhow!("decoded range shorter than requested span"))?;
Ok(slice.to_vec())
}
async fn throwaway_open(
client_ep: &iroh::Endpoint,
target: EndpointAddr,
client_node_id: B256,
client_eth: &Arc<PrivateKeySigner>,
pool_id: B256,
hash: Hash,
) -> anyhow::Result<u64> {
let (conn, total) =
idle_open(client_ep, target, client_node_id, client_eth, pool_id, hash).await?;
conn.close(0u32.into(), b"client-abandoned");
Ok(total)
}
async fn idle_open(
client_ep: &iroh::Endpoint,
target: EndpointAddr,
client_node_id: B256,
client_eth: &Arc<PrivateKeySigner>,
pool_id: B256,
hash: Hash,
) -> anyhow::Result<(iroh::endpoint::Connection, u64)> {
let conn = client_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("open_bi: {e}"))?;
let binding_hash =
binding_signing_hash(client_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = client_eth
.sign_hash_sync(&binding_hash)?
.as_bytes()
.to_vec();
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: client_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset: 0,
byte_len: 0,
timestamp_us: 0x9002,
};
let payload =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame_to(&mut send, &payload).await?;
let (resp, resp_ext) = read_stream_response(&mut recv).await?;
anyhow::ensure!(resp.body.ok, "idle open refused: {:?}", resp_ext.error);
std::mem::forget(send);
std::mem::forget(recv);
Ok((conn, resp.body.total_bytes))
}
async fn write_frame_to(send: &mut SendStream, payload: &[u8]) -> anyhow::Result<()> {
write_frame(send, payload)
.await
.map_err(|e| anyhow::anyhow!("write frame: {e}"))
}
async fn mount_range_probe(server: &MockServer, hex: &str, blob: &[u8]) {
let end = blob.len().min(16 * 1024);
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", probe_range_val(blob.len()).as_str()))
.respond_with(
ResponseTemplate::new(206).set_body_bytes(blob.get(..end).unwrap_or_default().to_vec()),
)
.mount(server)
.await;
}
fn probe_range_val(len: usize) -> String {
format!("bytes=0-{}", len.min(16 * 1024).saturating_sub(1))
}
async fn count_requests(
server: &MockServer,
pred: impl Fn(&wiremock::Request) -> bool,
) -> anyhow::Result<usize> {
let reqs = server
.received_requests()
.await
.ok_or_else(|| anyhow::anyhow!("wiremock request recording disabled"))?;
Ok(reqs.iter().filter(|r| pred(r)).count())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)] async fn cold_range_request_pulls_only_the_range_from_origin() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let (req_off, req_len) = (16 * 1024u64, 32 * 1024u64);
let aligned = align_range(req_off, req_len, blob_size)?;
let (a_start, a_end) = (aligned.fetch_start(), aligned.fetch_end());
let span = blob
.get(usize::try_from(a_start)?..usize::try_from(a_end)?)
.ok_or_else(|| anyhow::anyhow!("aligned span out of bounds"))?
.to_vec();
let range_val = format!("bytes={a_start}-{}", a_end - 1);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(ResponseTemplate::new(206).set_body_bytes(span.clone()))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x42);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
req_off,
req_len,
RATE_PER_MB,
)
.await?;
let want = blob
.get(usize::try_from(req_off)?..usize::try_from(req_off + req_len)?)
.ok_or_else(|| anyhow::anyhow!("requested range out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"delivered range mismatch: got {} bytes, want {}",
got.len(),
want.len()
);
let probe = probe_range_val(blob.len());
let ranged_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& r.headers
.get("range")
.is_some_and(|v| v.to_str().is_ok_and(|v| v != probe))
})
.await?;
let wholeblob_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& !r.headers.contains_key("range")
})
.await?;
anyhow::ensure!(
ranged_gets == 1,
"expected one ranged data GET, got {ranged_gets}"
);
anyhow::ensure!(
wholeblob_gets == 0,
"the whole blob must never be fetched to serve a range, saw {wholeblob_gets} un-ranged GET(s)"
);
anyhow::ensure!(
!cache.has(hash).await?,
"a range pull must leave the blob partial, not a full holder"
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"a bounded cold miss must take the own-origin two-leg tier"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn truncated_outboard_declines_before_signing_and_falls_back() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let truncated = outboard
.get(..outboard.len().saturating_sub(7))
.ok_or_else(|| anyhow::anyhow!("outboard too short for the fixture"))?
.to_vec();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(truncated))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(blob.clone()))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x68);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
Some(Duration::from_secs(15)),
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
0,
0,
RATE_PER_MB,
)
.await?;
anyhow::ensure!(
got.as_slice() == blob.as_slice(),
"the buffered fallback must deliver the whole blob byte-exact"
);
anyhow::ensure!(cache.has(hash).await?, "the fallback caches the blob");
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 0,
"a truncated outboard must not enter the two-leg tier"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn range_request_without_outboard_falls_back_to_whole_blob() -> anyhow::Result<()> {
let (blob, _outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let (req_off, req_len) = (16 * 1024u64, 32 * 1024u64);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(404))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(blob.clone()))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x43);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, _metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
Some(Duration::from_secs(15)),
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
req_off,
req_len,
RATE_PER_MB,
)
.await?;
let want = blob
.get(usize::try_from(req_off)?..usize::try_from(req_off + req_len)?)
.ok_or_else(|| anyhow::anyhow!("requested range out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"fallback must still deliver the correct range bytes"
);
anyhow::ensure!(
cache.has(hash).await?,
"the whole-blob fallback must complete and cache the blob"
);
let hex = hash.to_hex();
let ranged_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& r.headers.contains_key("range")
})
.await?;
let wholeblob_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& !r.headers.contains_key("range")
})
.await?;
anyhow::ensure!(
ranged_gets == 0,
"no ranged GET should occur once the outboard probe misses, saw {ranged_gets}"
);
anyhow::ensure!(
wholeblob_gets >= 1,
"the fallback must fetch the whole blob with an un-ranged GET"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn unauthorized_range_request_triggers_no_origin_fetch() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x44);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let _provider = server_eth.address();
let (handler, cache, _metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let conn = client_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("open_bi: {e}"))?;
let ext = StreamRequestExt {
binding: None,
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset: 16 * 1024,
byte_len: 32 * 1024,
timestamp_us: 0x9001,
};
let payload =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame_to(&mut send, &payload).await?;
match read_client_msg(&mut recv).await? {
ClientMessage::StreamResponse(r) => {
anyhow::ensure!(!r.body.ok, "unauthorized range request must be refused");
}
other => anyhow::bail!("expected a refusing StreamResponse, got {other:?}"),
}
conn.close(0u32.into(), b"done");
let origin_hits = count_requests(&server, |_| true).await?;
anyhow::ensure!(
origin_hits == 0,
"an unauthorized range request must not front any origin egress, saw {origin_hits} request(s)"
);
anyhow::ensure!(
!cache.has(hash).await?,
"no blob should have been cached for an unauthorized request"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn resume_to_end_range_pull_serves_tail() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let req_off = 16 * 1024u64;
let aligned = align_range(req_off, 0, blob_size)?;
let (a_start, a_end) = (aligned.fetch_start(), aligned.fetch_end());
anyhow::ensure!(a_end == blob_size, "a zero-len tail aligns to the blob end");
let span = blob
.get(usize::try_from(a_start)?..usize::try_from(a_end)?)
.ok_or_else(|| anyhow::anyhow!("aligned tail out of bounds"))?
.to_vec();
let range_val = format!("bytes={a_start}-{}", a_end - 1);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(ResponseTemplate::new(206).set_body_bytes(span))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x46);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
req_off,
0, RATE_PER_MB,
)
.await?;
let want = blob
.get(usize::try_from(req_off)?..)
.ok_or_else(|| anyhow::anyhow!("offset past blob"))?;
anyhow::ensure!(
got.as_slice() == want,
"a zero-len resume must deliver the whole tail, got {} of {} bytes",
got.len(),
want.len()
);
anyhow::ensure!(
!cache.has(hash).await?,
"a tail range pull leaves the blob partial"
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"a resumed cold miss must take the own-origin two-leg tier"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn end_past_the_blob_own_origin_cold_miss_serves_clamped_tail() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let req_off = 16 * 1024u64;
let aligned = align_range(req_off, 0, blob_size)?;
let (a_start, a_end) = (aligned.fetch_start(), aligned.fetch_end());
anyhow::ensure!(a_end == blob_size, "a clamped tail aligns to the blob end");
let span = blob
.get(usize::try_from(a_start)?..usize::try_from(a_end)?)
.ok_or_else(|| anyhow::anyhow!("aligned tail out of bounds"))?
.to_vec();
let range_val = format!("bytes={a_start}-{}", a_end - 1);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(ResponseTemplate::new(206).set_body_bytes(span))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x49);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
req_off,
blob_size, RATE_PER_MB,
)
.await?;
let want = blob
.get(usize::try_from(req_off)?..)
.ok_or_else(|| anyhow::anyhow!("offset past blob"))?;
anyhow::ensure!(
got.as_slice() == want,
"an end past the blob must deliver clamped to the blob's end through the \
own-origin cold-miss tier, got {} of {} bytes",
got.len(),
want.len()
);
anyhow::ensure!(
!cache.has(hash).await?,
"a tail range pull leaves the blob partial"
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"a clamped cold miss must take the own-origin two-leg tier"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn offset_past_the_blob_is_rejected_before_delivery() -> anyhow::Result<()> {
let (blob, _outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(blob.clone()))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x47);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let _provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
cache.populate(hash).await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let conn = client_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("open_bi: {e}"))?;
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset: blob_size,
byte_len: 0,
timestamp_us: 0x9001,
};
let payload = encode_stream_request(&req, Some(&StreamRequestExt::default()))
.map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame_to(&mut send, &payload).await?;
match read_client_msg(&mut recv).await? {
ClientMessage::StreamResponse(r) => {
anyhow::ensure!(
!r.body.ok,
"an offset at or past the blob end must be refused, not served"
);
}
other => anyhow::bail!("expected a refusing StreamResponse, got {other:?}"),
}
conn.close(0u32.into(), b"done");
anyhow::ensure!(
counter_value(
&metrics,
"serve_stream_rejected_range_not_satisfiable_total"
)? == 1,
"the out-of-bounds reject must increment the range-not-satisfiable counter"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn end_past_the_blob_is_served_clamped_not_refused() -> anyhow::Result<()> {
let (blob, _outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(blob.clone()))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x48);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
cache.populate(hash).await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let req_off = 16 * 1024u64;
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
req_off,
blob_size,
RATE_PER_MB,
)
.await?;
let want = blob
.get(usize::try_from(req_off)?..)
.ok_or_else(|| anyhow::anyhow!("offset past blob"))?;
anyhow::ensure!(
got.as_slice() == want,
"an end past the blob must deliver clamped to the blob's end, got {} of {} bytes",
got.len(),
want.len()
);
anyhow::ensure!(
counter_value(
&metrics,
"serve_stream_rejected_range_not_satisfiable_total"
)? == 0,
"a clamped end must not fire the range-not-satisfiable reject path"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
fn parse_byte_range(h: &str) -> Option<(u64, u64)> {
let (a, b) = h.strip_prefix("bytes=")?.split_once('-')?;
Some((a.parse().ok()?, b.parse().ok()?))
}
#[tokio::test(flavor = "multi_thread")]
async fn whole_blob_own_origin_miss_serves_via_backend_origin() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let aligned = align_range(0, 0, blob_size)?;
let (a_start, a_end) = (aligned.fetch_start(), aligned.fetch_end());
let span = blob
.get(usize::try_from(a_start)?..usize::try_from(a_end)?)
.ok_or_else(|| anyhow::anyhow!("aligned span out of bounds"))?
.to_vec();
let range_val = format!("bytes={a_start}-{}", a_end - 1);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(ResponseTemplate::new(206).set_body_bytes(span.clone()))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x51);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, _cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
0,
0,
RATE_PER_MB,
)
.await?;
anyhow::ensure!(
got.as_slice() == blob.as_slice(),
"whole-blob own-origin delivery mismatch: got {} bytes, want {}",
got.len(),
blob.len()
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"the own-origin two-leg serve tier must fire once"
);
anyhow::ensure!(
counter_value(&metrics, "serve_cache_miss_total")? == 1
&& counter_value(&metrics, "serve_cache_hit_total")? == 0
&& counter_value(&metrics, "serve_first_byte_miss_seconds_count")? == 1
&& counter_value(&metrics, "serve_first_byte_hit_seconds_count")? == 0,
"a serve that ran a fill tier must be classified a miss, not a hit"
);
let obao_gets = count_requests(&server, |r| {
r.method.as_str() == "GET" && r.url.path() == format!("/{hex}.obao4")
})
.await?;
anyhow::ensure!(
obao_gets == 1,
"the outboard must be read from the origin once, read {obao_gets} times"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
anyhow::ensure!(
counter_value(&metrics, "bytes_received_total")? == 0,
"an own-origin fill must not count as bytes received from peers"
);
anyhow::ensure!(
counter_value(&metrics, "bytes_served_total")? > 0,
"the served blob must count as bytes served"
);
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn bounded_whole_blob_own_origin_miss_streams_via_backend_origin() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let aligned = align_range(0, blob_size, blob_size)?;
let (a_start, a_end) = (aligned.fetch_start(), aligned.fetch_end());
let span = blob
.get(usize::try_from(a_start)?..usize::try_from(a_end)?)
.ok_or_else(|| anyhow::anyhow!("aligned span out of bounds"))?
.to_vec();
let range_val = format!("bytes={a_start}-{}", a_end - 1);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(ResponseTemplate::new(206).set_body_bytes(span.clone()))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x62);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, _cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
0,
blob_size,
RATE_PER_MB,
)
.await?;
anyhow::ensure!(
got.as_slice() == blob.as_slice(),
"bounded whole-blob delivery mismatch: got {} bytes, want {}",
got.len(),
blob.len()
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"a bounded whole-blob miss must take the own-origin two-leg tier"
);
let wholeblob_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& !r.headers.contains_key("range")
})
.await?;
anyhow::ensure!(
wholeblob_gets == 0,
"the spine pulls ranged spans only, saw {wholeblob_gets} un-ranged GET(s)"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn bounded_unaligned_offset_own_origin_miss_streams_the_exact_bytes() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let (req_off, req_len) = (20 * 1024u64, 30 * 1024u64);
let aligned = align_range(req_off, req_len, blob_size)?;
let (a_start, a_end) = (aligned.fetch_start(), aligned.fetch_end());
anyhow::ensure!(
a_start == 16 * 1024 && a_end == 64 * 1024,
"test premise: aligned span"
);
let span = blob
.get(usize::try_from(a_start)?..usize::try_from(a_end)?)
.ok_or_else(|| anyhow::anyhow!("aligned span out of bounds"))?
.to_vec();
let range_val = format!("bytes={a_start}-{}", a_end - 1);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(ResponseTemplate::new(206).set_body_bytes(span.clone()))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x63);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, _cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
req_off,
req_len,
RATE_PER_MB,
)
.await?;
let want = blob
.get(usize::try_from(req_off)?..usize::try_from(req_off + req_len)?)
.ok_or_else(|| anyhow::anyhow!("requested range out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"unaligned range mismatch: got {} bytes, want {}",
got.len(),
want.len()
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"an unaligned bounded miss must take the own-origin two-leg tier"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn throwaway_open_then_real_open_shares_one_multi_draw_fill() -> anyhow::Result<()> {
let (blob, outboard, hash) = large_blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
let blob_for_resp = blob.clone();
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header_exists("range"))
.respond_with(move |req: &Request| {
let span = req
.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
.and_then(|(s, e)| Some((usize::try_from(s).ok()?, usize::try_from(e).ok()?)))
.and_then(|(s, e)| blob_for_resp.get(s..=e));
match span {
Some(body) => ResponseTemplate::new(206)
.set_body_bytes(body.to_vec())
.set_delay(Duration::from_millis(300)),
None => ResponseTemplate::new(416),
}
})
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x64);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let total = throwaway_open(
&client_ep,
target.clone(),
client_node_id,
&client_eth,
pool_id,
hash,
)
.await?;
anyhow::ensure!(total == blob_size, "throwaway header total mismatch");
let got = tokio::time::timeout(
Duration::from_secs(30),
ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
0,
total,
RATE_PER_MB,
),
)
.await
.map_err(|_| anyhow::anyhow!("real open hung behind the throwaway's fill"))??;
anyhow::ensure!(
got.as_slice() == blob.as_slice(),
"real open after a throwaway must deliver byte-exact"
);
anyhow::ensure!(
cache.has(hash).await?,
"the real client paid the fill to the end"
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 2,
"both the throwaway and the real open must enter the two-leg tier"
);
let wholeblob_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& !r.headers.contains_key("range")
})
.await?;
anyhow::ensure!(wholeblob_gets == 0, "saw {wholeblob_gets} un-ranged GET(s)");
let reqs = server
.received_requests()
.await
.ok_or_else(|| anyhow::anyhow!("wiremock request recording disabled"))?;
let ranges: Vec<String> = reqs
.iter()
.filter(|r| r.method.as_str() == "GET" && r.url.path() == format!("/{hex}"))
.filter_map(|r| r.headers.get("range").and_then(|v| v.to_str().ok()))
.map(str::to_owned)
.collect();
anyhow::ensure!(
ranges.len() >= 2,
"test premise: a 3 MiB blob takes several draws, saw {ranges:?}"
);
let mut counts = std::collections::BTreeMap::<&str, usize>::new();
for range in &ranges {
*counts.entry(range.as_str()).or_default() += 1;
}
let repeated: Vec<(&str, usize)> = counts.into_iter().filter(|&(_, n)| n > 1).collect();
let probe = probe_range_val(blob.len());
let first_draw_repeated_once = |&(range, n): &(&str, usize)| {
n == 2 && range != probe && parse_byte_range(range).is_some_and(|(start, _)| start == 0)
};
anyhow::ensure!(
repeated.len() <= 1 && repeated.iter().all(first_draw_repeated_once),
"a span other than the aborted first draw was fetched twice in full: {ranges:?}"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn tail_open_ahead_of_a_stalled_owner_does_not_starve() -> anyhow::Result<()> {
let (blob, outboard, hash) = large_blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let tail_off = 2 * 1024 * 1024u64;
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
let blob_for_resp = blob.clone();
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header_exists("range"))
.respond_with(move |req: &Request| {
let span = req
.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
.and_then(|(s, e)| Some((usize::try_from(s).ok()?, usize::try_from(e).ok()?)))
.and_then(|(s, e)| blob_for_resp.get(s..=e));
match span {
Some(body) => ResponseTemplate::new(206)
.set_body_bytes(body.to_vec())
.set_delay(Duration::from_millis(200)),
None => ResponseTemplate::new(416),
}
})
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x66);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, _cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server_concurrent(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let (stalled_conn, total) = idle_open(
&client_ep,
target.clone(),
client_node_id,
&client_eth,
pool_id,
hash,
)
.await?;
anyhow::ensure!(total == blob_size, "idle open header total mismatch");
tokio::time::sleep(Duration::from_millis(600)).await;
let started = std::time::Instant::now();
let got = tokio::time::timeout(
Duration::from_secs(15),
ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
tail_off,
0,
RATE_PER_MB,
),
)
.await
.map_err(|_| anyhow::anyhow!("the tail open starved behind the throwaway's parked fill"))??;
let want = blob
.get(usize::try_from(tail_off)?..)
.ok_or_else(|| anyhow::anyhow!("tail out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"tail delivery mismatch: got {} bytes, want {}",
got.len(),
want.len()
);
anyhow::ensure!(
started.elapsed() < Duration::from_secs(10),
"the tail must not wait on a stalled fill: took {:?}",
started.elapsed()
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 2,
"both the stalled open and the tail open must enter the two-leg tier"
);
stalled_conn.close(0u32.into(), b"done");
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn throwaway_open_torn_down_before_real_open_still_serves() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
let blob_for_resp = blob.clone();
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header_exists("range"))
.respond_with(move |req: &Request| {
let span = req
.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
.and_then(|(s, e)| Some((usize::try_from(s).ok()?, usize::try_from(e).ok()?)))
.and_then(|(s, e)| blob_for_resp.get(s..=e));
match span {
Some(body) => ResponseTemplate::new(206).set_body_bytes(body.to_vec()),
None => ResponseTemplate::new(416),
}
})
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x65);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, _cache, _metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let total = throwaway_open(
&client_ep,
target.clone(),
client_node_id,
&client_eth,
pool_id,
hash,
)
.await?;
tokio::time::sleep(Duration::from_millis(300)).await;
let got = tokio::time::timeout(
Duration::from_secs(20),
ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
0,
total,
RATE_PER_MB,
),
)
.await
.map_err(|_| anyhow::anyhow!("real open hung after the throwaway's teardown"))??;
anyhow::ensure!(
got.as_slice() == blob.as_slice(),
"real open after a torn-down throwaway must deliver byte-exact"
);
let wholeblob_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& !r.headers.contains_key("range")
})
.await?;
anyhow::ensure!(wholeblob_gets == 0, "saw {wholeblob_gets} un-ranged GET(s)");
let ranged_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& r.headers.contains_key("range")
})
.await?;
anyhow::ensure!(
ranged_gets <= 2,
"a torn-down throwaway costs at most one duplicate draw, saw {ranged_gets}"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn no_range_origin_degrades_to_buffered_before_signing() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(blob.clone()))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x6B);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
Some(Duration::from_secs(15)),
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = tokio::time::timeout(
Duration::from_secs(20),
ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
0,
0,
RATE_PER_MB,
),
)
.await
.map_err(|_| anyhow::anyhow!("a no-Range origin must degrade, not hang"))??;
anyhow::ensure!(
got.as_slice() == blob.as_slice(),
"the buffered fallback must deliver the whole blob byte-exact"
);
anyhow::ensure!(cache.has(hash).await?, "the fallback caches the blob");
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 0,
"the range probe must decline before the two-leg tier signs"
);
let obao_gets = count_requests(&server, |r| r.url.path() == format!("/{hex}.obao4")).await?;
anyhow::ensure!(obao_gets == 1, "one outboard read, got {obao_gets}");
let probe = probe_range_val(blob.len());
let probe_gets = count_requests(&server, |r| {
r.url.path() == format!("/{hex}")
&& r.headers
.get("range")
.is_some_and(|v| v.to_str().is_ok_and(|v| v == probe))
})
.await?;
anyhow::ensure!(probe_gets == 1, "one probe range read, got {probe_gets}");
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn own_origin_size_probe_fault_refuses_internal_error_not_cache_miss() -> anyhow::Result<()> {
let (blob, _outboard, hash) = blob_with_outboard();
let hex = hash.to_hex();
drop(blob);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(ResponseTemplate::new(500))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x67);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let (handler, _cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let conn = client_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("open_bi: {e}"))?;
let binding_hash =
binding_signing_hash(client_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = client_eth
.sign_hash_sync(&binding_hash)?
.as_bytes()
.to_vec();
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: client_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset: 0,
byte_len: 0,
timestamp_us: 0x9003,
};
let payload =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame_to(&mut send, &payload).await?;
let (resp, _resp_ext) = read_stream_response(&mut recv).await?;
anyhow::ensure!(!resp.body.ok, "a faulted node must refuse");
conn.close(0u32.into(), b"done");
anyhow::ensure!(
counter_value(&metrics, "serve_stream_rejected_internal_error_total")? == 1,
"a size-probe transport fault must refuse as InternalError (degraded node)"
);
anyhow::ensure!(
counter_value(&metrics, "serve_stream_rejected_cache_miss_total")? == 0,
"a degraded node must not be reported as an empty one"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn own_origin_range_probe_fault_refuses_internal_error_not_cache_miss() -> anyhow::Result<()>
{
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", probe_range_val(blob.len()).as_str()))
.respond_with(ResponseTemplate::new(206).set_delay(Duration::from_secs(5)))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x6C);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
cache.set_origin_read_budget(Duration::from_millis(500), 1_000_000);
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let conn = client_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("open_bi: {e}"))?;
let binding_hash =
binding_signing_hash(client_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = client_eth
.sign_hash_sync(&binding_hash)?
.as_bytes()
.to_vec();
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: client_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset: 0,
byte_len: 0,
timestamp_us: 0x9004,
};
let payload =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame_to(&mut send, &payload).await?;
let (resp, _resp_ext) = read_stream_response(&mut recv).await?;
anyhow::ensure!(!resp.body.ok, "a faulted node must refuse");
conn.close(0u32.into(), b"done");
anyhow::ensure!(
counter_value(&metrics, "serve_stream_rejected_internal_error_total")? == 1,
"a range-probe transport fault must refuse as InternalError (degraded node)"
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 0,
"the two-leg tier must not be entered after a faulted probe"
);
anyhow::ensure!(
counter_value(&metrics, "serve_stream_rejected_cache_miss_total")? == 0,
"a degraded node must not be reported as an empty one"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn cold_out_of_bounds_range_is_refused_before_signing() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let aligned = align_range(0, 0, blob_size)?;
let span = blob.clone();
let range_val = format!(
"bytes={}-{}",
aligned.fetch_start(),
aligned.fetch_end() - 1
);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(ResponseTemplate::new(206).set_body_bytes(span))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x69);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, _cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let conn = client_ep
.connect(target.clone(), ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("open_bi: {e}"))?;
let binding_hash =
binding_signing_hash(client_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = client_eth
.sign_hash_sync(&binding_hash)?
.as_bytes()
.to_vec();
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: client_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset: blob_size,
byte_len: 16 * 1024,
timestamp_us: 0x9004,
};
let payload =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame_to(&mut send, &payload).await?;
let (resp, _resp_ext) = read_stream_response(&mut recv).await?;
anyhow::ensure!(!resp.body.ok, "an out-of-bounds cold miss must be refused");
conn.close(0u32.into(), b"done");
anyhow::ensure!(
counter_value(
&metrics,
"serve_stream_rejected_range_not_satisfiable_total"
)? == 1,
"the spine's pre-signature gate must answer RangeNotSatisfiable"
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"the gate lives inside the spine tier (entered, then refused pre-signature)"
);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
0,
0,
RATE_PER_MB,
)
.await?;
anyhow::ensure!(
got.as_slice() == blob.as_slice(),
"an in-bounds request after the refusal must still serve"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn span_capped_floor_accepts_a_small_range_a_window_poor_pool() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let (req_off, req_len) = (16 * 1024u64, 48 * 1024u64);
let aligned = align_range(req_off, req_len, blob_size)?;
let (a_start, a_end) = (aligned.fetch_start(), aligned.fetch_end());
let span = blob
.get(usize::try_from(a_start)?..usize::try_from(a_end)?)
.ok_or_else(|| anyhow::anyhow!("aligned span out of bounds"))?
.to_vec();
let range_val = format!("bytes={a_start}-{}", a_end - 1);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(ResponseTemplate::new(206).set_body_bytes(span))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x6A);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let store = Arc::new(MemoryPoolStateStore::new());
store.record(&LaneState::hydrate(
pool_id,
client_eth.address(),
provider,
U256::from(10_000_000u64),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
let cache_dir = tempfile::tempdir()?;
let origin = Arc::new(HttpOrigin::parse(&server.uri())?);
let cache = CacheEngine::open(cache_dir.path(), vec![origin as Arc<dyn Origin>], 16).await?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let store_dyn: Arc<dyn PoolStateStore> = store;
let mut status = std::collections::HashMap::new();
status.insert(
pool_id,
decdn_node::pool_view::PoolStatus {
owner: client_eth.address(),
remaining: U256::from(5u64),
lifecycle: decdn_node::pool_view::Lifecycle::Open,
},
);
let pool_view =
Arc::new(StatusMapPoolView { status }) as Arc<dyn decdn_node::pool_view::PoolView>;
let handler = build_handler_full_configured(
server_id,
&server_eth,
&metrics,
limiter,
cache,
store_dyn,
RATE_PER_MB,
&domains(),
16,
|deps| deps.pool_view = Some(pool_view),
)?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target.clone(),
client_node_id,
&client_eth,
pool_id,
provider,
hash,
req_off,
req_len,
RATE_PER_MB,
)
.await?;
let want = blob
.get(usize::try_from(req_off)?..usize::try_from(req_off + req_len)?)
.ok_or_else(|| anyhow::anyhow!("requested range out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"a span the pool can cover must be served"
);
anyhow::ensure!(
counter_value(&metrics, "serve_stream_rejected_insufficient_deposit_total")? == 0,
"the span-capped floor must not refuse a coverable range"
);
let conn = client_ep
.connect(target, ALPN_CLIENT)
.await
.map_err(|e| anyhow::anyhow!("connect: {e}"))?;
let (mut send, mut recv) = conn
.open_bi()
.await
.map_err(|e| anyhow::anyhow!("open_bi: {e}"))?;
let binding_hash =
binding_signing_hash(client_node_id, EPHEMERAL_BINDING_NONCE, &binding_dom());
let binding_signature = client_eth
.sign_hash_sync(&binding_hash)?
.as_bytes()
.to_vec();
let ext = StreamRequestExt {
binding: Some(ClientBinding {
ethereum_address: client_eth.address().into(),
binding_signature,
}),
capability: None,
};
let req = StreamRequest {
hash: *hash.as_bytes(),
namespace_id: decdn_protocol::client::NO_NAMESPACE,
pool_id: pool_id.into(),
byte_offset: 0,
byte_len: 0,
timestamp_us: 0x9005,
};
let payload =
encode_stream_request(&req, Some(&ext)).map_err(|e| anyhow::anyhow!("encode req: {e}"))?;
write_frame_to(&mut send, &payload).await?;
let (resp, _resp_ext) = read_stream_response(&mut recv).await?;
anyhow::ensure!(
!resp.body.ok,
"a whole-blob request against a window-poor pool must be refused"
);
conn.close(0u32.into(), b"done");
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[derive(Debug)]
struct StatusMapPoolView {
status: std::collections::HashMap<B256, decdn_node::pool_view::PoolStatus>,
}
#[async_trait::async_trait]
impl decdn_node::pool_view::PoolView for StatusMapPoolView {
async fn status(&self, pool_id: B256) -> Option<decdn_node::pool_view::PoolStatus> {
self.status.get(&pool_id).copied()
}
}
fn large_blob_with_outboard() -> (Vec<u8>, Vec<u8>, Hash) {
let blob: Vec<u8> = (0..3 * 1024 * 1024u32)
.map(|i| u8::try_from(i.wrapping_mul(2_654_435_761) >> 24).unwrap_or(0))
.collect();
let ob = PreOrderMemOutboard::create(&blob, IROH_BLOCK_SIZE);
let hash = Hash::from_bytes(*ob.root.as_bytes());
(blob, ob.data, hash)
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn interior_hold_own_origin_miss_pulls_only_the_gaps() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let held_off = 64 * 1024u64;
let held_len = 64 * 1024u64;
let held_aligned = align_range(held_off, held_len, blob_size)?;
let (held_start, held_end) = (held_aligned.fetch_start(), held_aligned.fetch_end());
let held_data = blob
.get(usize::try_from(held_start)?..usize::try_from(held_end)?)
.ok_or_else(|| anyhow::anyhow!("held span out of bounds"))?
.to_vec();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
let blob_for_resp = blob.clone();
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header_exists("range"))
.respond_with(move |req: &Request| {
let span = req
.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
.and_then(|(s, e)| Some((usize::try_from(s).ok()?, usize::try_from(e).ok()?)))
.and_then(|(s, e)| blob_for_resp.get(s..=e));
match span {
Some(body) => ResponseTemplate::new(206).set_body_bytes(body.to_vec()),
None => ResponseTemplate::new(416),
}
})
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x52);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let held_bao = encode_verified_range(
*hash.as_bytes(),
&held_aligned,
&held_data,
bytes::Bytes::from(outboard.clone()),
)
.map_err(|e| anyhow::anyhow!("encode held range: {e:?}"))?;
cache
.admit_bao(hash, held_aligned.chunk_ranges().clone(), held_bao)
.await?;
anyhow::ensure!(
!cache.present_ranges(hash).await?.is_complete(),
"the seeded interior range must leave the blob a partial, not complete"
);
anyhow::ensure!(
!cache.has(hash).await?,
"an interior partial must not read as a full holder"
);
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let got = ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
0,
0,
RATE_PER_MB,
)
.await?;
anyhow::ensure!(
got.as_slice() == blob.as_slice(),
"interior-hold whole-blob delivery mismatch: got {} bytes, want {}",
got.len(),
blob.len()
);
let held_overlap_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& r.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
.is_some_and(|(s, e)| s < held_end && e >= held_start)
})
.await?;
anyhow::ensure!(
held_overlap_gets == 0,
"the origin must never re-fetch the held interior range, saw {held_overlap_gets} \
overlapping ranged GET(s)"
);
let prefix_gap_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& r.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
.is_some_and(|(s, _e)| s < held_start)
})
.await?;
let suffix_gap_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& r.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
.is_some_and(|(s, _e)| s >= held_end)
})
.await?;
anyhow::ensure!(
prefix_gap_gets >= 1,
"the prefix gap [0, {held_start}) must be pulled from origin"
);
anyhow::ensure!(
suffix_gap_gets >= 1,
"the suffix gap [{held_end}, {blob_size}) must be pulled from origin"
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"the own-origin two-leg serve tier must fire once"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn a_bounded_miss_seeds_only_the_held_ranges_inside_it() -> anyhow::Result<()> {
const G: u64 = 16 * 1024;
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
let blob_for_resp = blob.clone();
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header_exists("range"))
.respond_with(move |req: &Request| {
let span = req
.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
.and_then(|(s, e)| Some((usize::try_from(s).ok()?, usize::try_from(e).ok()?)))
.and_then(|(s, e)| blob_for_resp.get(s..=e));
match span {
Some(body) => ResponseTemplate::new(206).set_body_bytes(body.to_vec()),
None => ResponseTemplate::new(416),
}
})
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x53);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let held = [(0, G), (4 * G, 4 * G), (12 * G, blob_size - 12 * G)];
for (off, len) in held {
let aligned = align_range(off, len, blob_size)?;
let data = blob
.get(usize::try_from(aligned.fetch_start())?..usize::try_from(aligned.fetch_end())?)
.ok_or_else(|| anyhow::anyhow!("held span out of bounds"))?
.to_vec();
let bao = encode_verified_range(
*hash.as_bytes(),
&aligned,
&data,
bytes::Bytes::from(outboard.clone()),
)
.map_err(|e| anyhow::anyhow!("encode held range: {e:?}"))?;
cache
.admit_bao(hash, aligned.chunk_ranges().clone(), bao)
.await?;
}
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let (req_off, req_len) = (3 * G + 100, 6 * G - 150);
let got = tokio::time::timeout(
Duration::from_secs(30),
ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
req_off,
req_len,
RATE_PER_MB,
),
)
.await
.map_err(|_| anyhow::anyhow!("the bounded miss over held ranges hung"))??;
let want = blob
.get(usize::try_from(req_off)?..usize::try_from(req_off + req_len)?)
.ok_or_else(|| anyhow::anyhow!("requested range out of bounds"))?;
anyhow::ensure!(
got.as_slice() == want,
"bounded miss over held ranges must be byte-exact"
);
let ranged_gets: Vec<(u64, u64)> = server
.received_requests()
.await
.ok_or_else(|| anyhow::anyhow!("wiremock request recording disabled"))?
.iter()
.filter(|r| r.method.as_str() == "GET" && r.url.path() == format!("/{hex}"))
.filter_map(|r| {
r.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
})
.collect();
for (s, e) in &ranged_gets {
anyhow::ensure!(
!(*s < 8 * G && *e >= 4 * G),
"the origin must never re-fetch held groups 4-7, saw bytes={s}-{e}"
);
}
for gap in [3 * G, 8 * G] {
anyhow::ensure!(
ranged_gets.iter().any(|(s, e)| *s <= gap && *e >= gap),
"gap group at {gap} must be pulled from origin, saw {ranged_gets:?}"
);
}
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"the own-origin two-leg serve tier must fire once"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
async fn own_origin_serve_fails_not_hangs_on_origin_fetch_error() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header_exists("range"))
.respond_with(ResponseTemplate::new(500))
.mount(&server)
.await;
let pool_id = B256::repeat_byte(0x53);
let client_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, _cache, metrics, _cache_tmp) = handler_over_http_origin(
&server.uri(),
pool_id,
client_eth.address(),
&server_eth,
server_id,
None,
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server(server_ep.clone(), handler);
let client_sk = fresh_key();
let client_node_id = B256::from(*client_sk.public().as_bytes());
let (client_ep, _) = local_endpoint(client_sk, vec![]).await?;
let target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let outcome = tokio::time::timeout(
Duration::from_secs(20),
ranged_paid_pull(
&client_ep,
target,
client_node_id,
&client_eth,
pool_id,
provider,
hash,
0,
0,
RATE_PER_MB,
),
)
.await;
let inner = outcome.map_err(|_| {
anyhow::anyhow!(
"own-origin serve HUNG on an origin fetch error — no terminal signal reached the client"
)
})?;
anyhow::ensure!(
inner.is_err(),
"an origin fetch failure must fail the serve; the client must not receive a clean whole blob"
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 1,
"the own-origin two-leg serve tier must have been selected before the origin fault"
);
shutdown([server_task], [&client_ep, &server_ep]).await?;
Ok(())
}
fn spawn_server_concurrent(
server_ep: iroh::Endpoint,
handler: Arc<ClientHandler>,
) -> tokio::task::JoinHandle<()> {
use decdn_node::handlers::client::ClientProtocol;
use iroh::protocol::ProtocolHandler;
tokio::spawn(async move {
while let Some(incoming) = server_ep.accept().await {
let Ok(connecting) = incoming.accept() else {
continue;
};
let Ok(conn) = connecting.await else { continue };
let handler = Arc::clone(&handler);
tokio::spawn(async move {
let _ = ClientProtocol::new(handler).accept(conn).await;
});
}
})
}
#[allow(clippy::too_many_arguments)]
async fn handler_two_pools_over_http_origin(
origin_uri: &str,
owner_pool: B256,
client_a: Address,
waiter_pool: B256,
client_b: Address,
server_eth: &Arc<PrivateKeySigner>,
server_id: iroh::PublicKey,
pull_through: Option<Duration>,
) -> anyhow::Result<(
Arc<ClientHandler>,
CacheEngine,
Arc<Metrics>,
tempfile::TempDir,
)> {
let store = Arc::new(MemoryPoolStateStore::new());
for (pool_id, client) in [(owner_pool, client_a), (waiter_pool, client_b)] {
store.record(&LaneState::hydrate(
pool_id,
client, server_eth.address(), U256::from(10_000_000u64),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
}
let cache_dir = tempfile::tempdir()?;
let origin = Arc::new(HttpOrigin::parse(origin_uri)?);
let cache = CacheEngine::open(cache_dir.path(), vec![origin as Arc<dyn Origin>], 16).await?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let store_dyn: Arc<dyn PoolStateStore> = store;
let handler = build_handler_full_configured(
server_id,
server_eth,
&metrics,
limiter,
cache.clone(),
store_dyn,
RATE_PER_MB,
&domains(),
16,
|deps| deps.pull_through = pull_through,
)?;
Ok((handler, cache, metrics, cache_dir))
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn mid_blob_open_behind_a_paying_owners_frontier_attaches() -> anyhow::Result<()> {
let (blob, outboard, hash) = large_blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
let blob_for_resp = blob.clone();
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header_exists("range"))
.respond_with(move |req: &Request| {
let span = req
.headers
.get("range")
.and_then(|v| v.to_str().ok())
.and_then(parse_byte_range)
.and_then(|(s, e)| Some((usize::try_from(s).ok()?, usize::try_from(e).ok()?)))
.and_then(|(s, e)| blob_for_resp.get(s..=e));
match span {
Some(body) => ResponseTemplate::new(206)
.set_body_bytes(body.to_vec())
.set_delay(Duration::from_millis(300)),
None => ResponseTemplate::new(416),
}
})
.mount(&server)
.await;
let owner_pool = B256::repeat_byte(0x6C);
let waiter_pool = B256::repeat_byte(0x6D);
let owner_eth = Arc::new(PrivateKeySigner::random());
let waiter_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let store = Arc::new(MemoryPoolStateStore::new());
for (pool_id, client) in [
(owner_pool, owner_eth.address()),
(waiter_pool, waiter_eth.address()),
] {
store.record(&LaneState::hydrate(
pool_id,
client,
server_eth.address(),
U256::from(10_000_000u64),
0,
U256::ZERO,
U256::ZERO,
None,
decdn_incentive::LaneChain::NONE,
))?;
}
let cache_dir = tempfile::tempdir()?;
let origin = Arc::new(HttpOrigin::parse(&server.uri())?);
let cache_metrics = Arc::new(decdn_cache::CacheMetrics::default());
let cache = CacheEngine::open_full(
cache_dir.path(),
vec![origin as Arc<dyn Origin>],
16,
decdn_cache::PinnedHashes::default(),
decdn_cache::RetryPolicy::default(),
decdn_cache::CircuitBreakerPolicy::default(),
Some(Arc::clone(&cache_metrics)),
Duration::ZERO,
)
.await?;
let metrics = Arc::new(Metrics::new());
let limiter = permissive_limiter(&metrics);
let store_dyn: Arc<dyn PoolStateStore> = store;
let handler = build_handler_full_configured(
server_id,
&server_eth,
&metrics,
limiter,
cache.clone(),
store_dyn,
RATE_PER_MB,
&domains(),
16,
|_| {},
)?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server_concurrent(server_ep.clone(), handler);
let owner_sk = fresh_key();
let owner_node_id = B256::from(*owner_sk.public().as_bytes());
let (owner_ep, _) = local_endpoint(owner_sk, vec![]).await?;
let waiter_sk = fresh_key();
let waiter_node_id = B256::from(*waiter_sk.public().as_bytes());
let (waiter_ep, _) = local_endpoint(waiter_sk, vec![]).await?;
let owner_target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let waiter_target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let owner_hash = hash;
let owner_eth2 = Arc::clone(&owner_eth);
let owner_ep_task = owner_ep.clone();
let a = tokio::spawn(async move {
ranged_paid_pull(
&owner_ep_task,
owner_target,
owner_node_id,
&owner_eth2,
owner_pool,
provider,
owner_hash,
0,
0,
RATE_PER_MB,
)
.await
});
let first_voucher = async {
while counter_value(&metrics, "vouchers_received_total")? == 0 {
anyhow::ensure!(!a.is_finished(), "the owner ended before its first voucher");
tokio::time::sleep(Duration::from_millis(10)).await;
}
anyhow::Ok(())
};
tokio::time::timeout(Duration::from_secs(30), first_voucher)
.await
.map_err(|_| anyhow::anyhow!("the owner's first voucher never landed"))??;
let (req_off, req_len) = (16 * 1024u64, 32 * 1024u64);
let got_waiter = ranged_paid_pull(
&waiter_ep,
waiter_target,
waiter_node_id,
&waiter_eth,
waiter_pool,
provider,
hash,
req_off,
req_len,
RATE_PER_MB,
)
.await?;
let got_owner = a.await??;
anyhow::ensure!(got_owner.as_slice() == blob.as_slice(), "owner byte-exact");
let want = blob
.get(usize::try_from(req_off)?..usize::try_from(req_off + req_len)?)
.ok_or_else(|| anyhow::anyhow!("requested range out of bounds"))?;
anyhow::ensure!(got_waiter.as_slice() == want, "attached range byte-exact");
anyhow::ensure!(cache.has(hash).await?, "the owner's fill completes");
anyhow::ensure!(
cache_metrics.fill_not_coalesced.get() == 0,
"a request behind the paying owner's frontier must attach, not own"
);
let reqs = server
.received_requests()
.await
.ok_or_else(|| anyhow::anyhow!("wiremock request recording disabled"))?;
let ranges: Vec<String> = reqs
.iter()
.filter(|r| r.method.as_str() == "GET" && r.url.path() == format!("/{hex}"))
.filter_map(|r| r.headers.get("range").and_then(|v| v.to_str().ok()))
.map(str::to_owned)
.collect();
let mut distinct = ranges.clone();
distinct.sort();
distinct.dedup();
anyhow::ensure!(
distinct.len() == ranges.len(),
"the overlap must not be fetched twice: {ranges:?}"
);
shutdown([server_task], [&owner_ep, &waiter_ep, &server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn concurrent_whole_blob_own_origin_misses_coalesce_to_one_pull() -> anyhow::Result<()> {
let (blob, outboard, hash) = blob_with_outboard();
let blob_size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let aligned = align_range(0, 0, blob_size)?;
let (a_start, a_end) = (aligned.fetch_start(), aligned.fetch_end());
let span = blob
.get(usize::try_from(a_start)?..usize::try_from(a_end)?)
.ok_or_else(|| anyhow::anyhow!("aligned span out of bounds"))?
.to_vec();
let range_val = format!("bytes={a_start}-{}", a_end - 1);
let server = MockServer::start().await;
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", blob_size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
mount_range_probe(&server, &hex, &blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(
ResponseTemplate::new(206)
.set_body_bytes(span.clone())
.set_delay(Duration::from_millis(1_200)),
)
.mount(&server)
.await;
let owner_pool = B256::repeat_byte(0x61);
let waiter_pool = B256::repeat_byte(0x62);
let owner_eth = Arc::new(PrivateKeySigner::random());
let waiter_eth = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_two_pools_over_http_origin(
&server.uri(),
owner_pool,
owner_eth.address(),
waiter_pool,
waiter_eth.address(),
&server_eth,
server_id,
Some(Duration::from_secs(10)),
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server_concurrent(server_ep.clone(), handler);
let owner_sk = fresh_key();
let owner_node_id = B256::from(*owner_sk.public().as_bytes());
let (owner_ep, _) = local_endpoint(owner_sk, vec![]).await?;
let waiter_sk = fresh_key();
let waiter_node_id = B256::from(*waiter_sk.public().as_bytes());
let (waiter_ep, _) = local_endpoint(waiter_sk, vec![]).await?;
let owner_target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let waiter_target = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let owner_hash = hash;
let a = tokio::spawn(async move {
ranged_paid_pull(
&owner_ep,
owner_target,
owner_node_id,
&owner_eth,
owner_pool,
provider,
owner_hash,
0,
0,
RATE_PER_MB,
)
.await
});
tokio::time::sleep(Duration::from_millis(400)).await;
let got_waiter = ranged_paid_pull(
&waiter_ep,
waiter_target,
waiter_node_id,
&waiter_eth,
waiter_pool,
provider,
hash,
0,
0,
RATE_PER_MB,
)
.await?;
let got_owner = a.await??;
anyhow::ensure!(
got_owner.as_slice() == blob.as_slice(),
"owner delivery mismatch: got {} bytes, want {}",
got_owner.len(),
blob.len()
);
anyhow::ensure!(
got_waiter.as_slice() == blob.as_slice(),
"coalesced delivery mismatch: got {} bytes, want {}",
got_waiter.len(),
blob.len()
);
let ranged_gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& r.headers
.get("range")
.and_then(|v| v.to_str().ok())
.is_some_and(|v| v == range_val)
})
.await?;
anyhow::ensure!(
ranged_gets == 1,
"expected exactly ONE origin ranged GET (coalesced), saw {ranged_gets}"
);
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 2,
"both misses run the own-origin two-leg serve tier (owner + attached observer)"
);
anyhow::ensure!(
cache.has(hash).await?,
"blob must be present after coalesced serve"
);
shutdown([server_task], [&server_ep]).await?;
Ok(())
}
#[tokio::test(flavor = "multi_thread")]
#[allow(clippy::too_many_lines)]
async fn two_concurrent_disjoint_own_origin_misses_two_fetches_no_wedge() -> anyhow::Result<()> {
let (blob_a, outboard_a, hash_a) = blob_with_outboard();
let blob_b: Vec<u8> = (0..190 * 1024u32)
.map(|i| u8::try_from((i % 251).wrapping_add(7)).unwrap_or(0))
.collect();
let ob_b = PreOrderMemOutboard::create(&blob_b, IROH_BLOCK_SIZE);
let hash_b = Hash::from_bytes(*ob_b.root.as_bytes());
let outboard_b = ob_b.data;
anyhow::ensure!(hash_a != hash_b, "the two blobs must have distinct hashes");
let server = MockServer::start().await;
let mut range_vals = Vec::new();
for (blob, outboard, hash) in [
(&blob_a, &outboard_a, hash_a),
(&blob_b, &outboard_b, hash_b),
] {
let size = u64::try_from(blob.len()).unwrap_or(u64::MAX);
let hex = hash.to_hex();
let aligned = align_range(0, 0, size)?;
let (a_start, a_end) = (aligned.fetch_start(), aligned.fetch_end());
let span = blob
.get(usize::try_from(a_start)?..usize::try_from(a_end)?)
.ok_or_else(|| anyhow::anyhow!("aligned span out of bounds"))?
.to_vec();
let range_val = format!("bytes={a_start}-{}", a_end - 1);
Mock::given(method("HEAD"))
.and(path(format!("/{hex}")))
.respond_with(
ResponseTemplate::new(200).insert_header("Content-Length", size.to_string()),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path(format!("/{hex}.obao4")))
.respond_with(ResponseTemplate::new(200).set_body_bytes(outboard.clone()))
.mount(&server)
.await;
mount_range_probe(&server, &hex, blob).await;
Mock::given(method("GET"))
.and(path(format!("/{hex}")))
.and(header("range", range_val.as_str()))
.respond_with(
ResponseTemplate::new(206)
.set_body_bytes(span)
.set_delay(Duration::from_millis(600)),
)
.mount(&server)
.await;
range_vals.push((hex, range_val));
}
let pool_a = B256::repeat_byte(0x71);
let pool_b = B256::repeat_byte(0x72);
let eth_a = Arc::new(PrivateKeySigner::random());
let eth_b = Arc::new(PrivateKeySigner::random());
let server_sk = fresh_key();
let server_id = server_sk.public();
let server_eth = Arc::new(PrivateKeySigner::random());
let provider = server_eth.address();
let (handler, cache, metrics, _cache_tmp) = handler_two_pools_over_http_origin(
&server.uri(),
pool_a,
eth_a.address(),
pool_b,
eth_b.address(),
&server_eth,
server_id,
Some(Duration::from_secs(10)),
)
.await?;
let (server_ep, server_addr) = local_endpoint(server_sk, vec![ALPN_CLIENT.to_vec()]).await?;
let server_task = spawn_server_concurrent(server_ep.clone(), handler);
let sk_a = fresh_key();
let node_a = B256::from(*sk_a.public().as_bytes());
let (ep_a, _) = local_endpoint(sk_a, vec![]).await?;
let sk_b = fresh_key();
let node_b = B256::from(*sk_b.public().as_bytes());
let (ep_b, _) = local_endpoint(sk_b, vec![]).await?;
let target_a = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let target_b = EndpointAddr::new(server_id).with_ip_addr(server_addr);
let first_signer = Arc::clone(ð_a);
let a = tokio::spawn(async move {
ranged_paid_pull(
&ep_a,
target_a,
node_a,
&first_signer,
pool_a,
provider,
hash_a,
0,
0,
RATE_PER_MB,
)
.await
});
let second_signer = Arc::clone(ð_b);
let b = tokio::spawn(async move {
ranged_paid_pull(
&ep_b,
target_b,
node_b,
&second_signer,
pool_b,
provider,
hash_b,
0,
0,
RATE_PER_MB,
)
.await
});
let (got_a, got_b) = tokio::time::timeout(Duration::from_secs(30), async {
anyhow::Ok((a.await??, b.await??))
})
.await
.map_err(|_| {
anyhow::anyhow!("disjoint concurrent serves did not complete — possible wedge")
})??;
anyhow::ensure!(
got_a.as_slice() == blob_a.as_slice(),
"hash A delivery mismatch: got {} bytes, want {}",
got_a.len(),
blob_a.len()
);
anyhow::ensure!(
got_b.as_slice() == blob_b.as_slice(),
"hash B delivery mismatch: got {} bytes, want {}",
got_b.len(),
blob_b.len()
);
for (hex, range_val) in &range_vals {
let gets = count_requests(&server, |r| {
r.method.as_str() == "GET"
&& r.url.path() == format!("/{hex}")
&& r.headers
.get("range")
.and_then(|v| v.to_str().ok())
.is_some_and(|v| v == range_val.as_str())
})
.await?;
anyhow::ensure!(
gets == 1,
"hash /{hex} must be fetched exactly once (own pull), saw {gets}"
);
}
anyhow::ensure!(
counter_value(&metrics, "local_outboard_serves_total")? == 2,
"both disjoint misses run the own-origin serve tier"
);
anyhow::ensure!(
cache.has(hash_a).await? && cache.has(hash_b).await?,
"both blobs must be present after their serves"
);
shutdown([server_task], [&server_ep]).await?;
Ok(())
}