use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use alloy::primitives::U256;
use alloy::providers::{DynProvider, Provider as _};
use alloy::signers::local::PrivateKeySigner;
use anyhow::Context as _;
use decdn_incentive::buyer_pool::BuyerPoolStore;
use decdn_incentive::payment_pool::PaymentPool;
use decdn_incentive::{slash_judge_domain, voucher_domain};
use iroh::{Endpoint, RelayUrl};
use serde::Deserialize;
use tokio_util::sync::CancellationToken;
use crate::buyer_pool::UnconfirmedOpens;
use crate::discovery::NodeCandidate;
use crate::funding::RunFunding;
use crate::ordered_writes::OrderedWrites;
use crate::session::{
ChainAdoption, ChainConfig, DepositPolicy, DiscoveryOptions, PinnedNode, PullSite,
};
use crate::stream_cap::LaneStreamCap;
use crate::{
Connections, CredentialSlot, FundingNeeded, LaneLedgers, PeerHealth, PoolReplaced,
ProgressClock, RecoveryGate, RunDemand, StopPolicy,
};
mod engine;
mod manifest_cache;
mod skip_cache;
#[doc(hidden)]
pub use crate::lane_sources::FaultContext;
pub const STAGING_DIR: &str = ".decdn-partial";
pub fn parse_hash(s: &str) -> anyhow::Result<[u8; 32]> {
let hex = s
.strip_prefix("0x")
.or_else(|| s.strip_prefix("b3:"))
.unwrap_or(s);
let h = blake3::Hash::from_hex(hex).map_err(|e| {
anyhow::anyhow!(
"invalid hash {s:?}: expected 64 hex chars (BLAKE3 digest), optional `0x`/`b3:` \
prefix: {e}"
)
})?;
Ok(*h.as_bytes())
}
#[derive(Debug, Clone, Deserialize)]
#[non_exhaustive]
pub struct BundleManifest {
pub entries: Vec<ManifestEntry>,
}
#[derive(Deserialize)]
struct ManifestV1 {
version: u32,
entries: Vec<ManifestEntry>,
}
impl BundleManifest {
#[must_use]
pub const fn new(entries: Vec<ManifestEntry>) -> Self {
Self { entries }
}
pub fn parse(bytes: &[u8]) -> anyhow::Result<Self> {
let manifest: ManifestV1 = serde_json::from_slice(bytes)
.map_err(|e| NotAManifest(format!("parse bundle JSON (expected v1 manifest): {e}")))?;
if manifest.version != 1 {
return Err(NotAManifest(format!(
"unsupported bundle version {} (this build supports v1)",
manifest.version
))
.into());
}
Ok(Self {
entries: manifest.entries,
})
}
pub fn cached(out_dir: &Path, hash: [u8; 32]) -> anyhow::Result<Option<Self>> {
manifest_cache::load(out_dir, hash)?
.map(|bytes| {
Self::parse(&bytes).with_context(|| {
format!(
"cached bundle manifest {}",
manifest_cache::cache_path(out_dir, hash).display()
)
})
})
.transpose()
}
#[must_use]
pub fn cache_path(out_dir: &Path, hash: [u8; 32]) -> PathBuf {
manifest_cache::cache_path(out_dir, hash)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct NotAManifest(String);
impl std::fmt::Display for NotAManifest {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for NotAManifest {}
#[derive(Debug, Clone, Deserialize)]
#[non_exhaustive]
pub struct ManifestEntry {
pub path: String,
pub hash: String,
#[serde(default)]
pub size: Option<u64>,
#[serde(default)]
pub chunks: Option<Vec<ManifestChunk>>,
}
impl ManifestEntry {
#[must_use]
pub fn new(path: impl Into<String>, hash: impl Into<String>, size: Option<u64>) -> Self {
Self {
path: path.into(),
hash: hash.into(),
size,
chunks: None,
}
}
#[must_use]
pub fn with_chunks(mut self, chunks: Vec<ManifestChunk>) -> Self {
self.chunks = Some(chunks);
self
}
}
#[derive(Debug, Clone, Deserialize)]
#[non_exhaustive]
pub struct ManifestChunk {
pub hash: String,
pub size: u64,
}
impl ManifestChunk {
#[must_use]
pub fn new(hash: impl Into<String>, size: u64) -> Self {
Self {
hash: hash.into(),
size,
}
}
}
#[derive(Debug)]
#[non_exhaustive]
pub enum Funding {
SelfOwned {
signer: Arc<PrivateKeySigner>,
deposit: DepositPolicy,
},
Delegated {
slot: CredentialSlot,
},
}
#[derive(Debug, Clone)]
pub struct NodeSelection {
pinned: Option<PinnedNode>,
registry: Option<Vec<NodeCandidate>>,
}
pub async fn select_nodes(
chain: &ChainConfig,
site: &PullSite,
discovery: &DiscoveryOptions,
) -> anyhow::Result<NodeSelection> {
if let Some(pinned) = discovery.pinned {
return Ok(NodeSelection {
pinned: Some(pinned),
registry: None,
});
}
Ok(NodeSelection {
pinned: None,
registry: Some(engine::discover_candidates(chain, site, discovery).await?),
})
}
pub trait BundleObserver: Send + Sync {
fn run_started(&self, download_total: Option<u64>, content_total: Option<u64>) {
let _ = (download_total, content_total);
}
fn file_started(
&self,
paths: &[String],
download_total: u64,
reconstruct_total: u64,
) -> Arc<dyn FileObserver> {
let _ = (paths, download_total, reconstruct_total);
Arc::new(NoFileObserver)
}
fn entry_landed(&self, landed: &EntryLanded) {
let _ = landed;
}
fn warning(&self, warning: &str) {
let _ = warning;
}
fn pool_replaced(&self, replaced: &PoolReplaced) {
let _ = replaced;
}
}
pub trait FileObserver: Send + Sync {
fn delivered(&self, received: u64, expected: u64) {
let _ = (received, expected);
}
fn discovering(&self) {}
fn pending(&self) {}
fn reconstructing(&self) {}
fn hashed(&self, bytes: u64) {
let _ = bytes;
}
fn finished(&self) {}
}
struct NoFileObserver;
impl FileObserver for NoFileObserver {}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct EntryLanded {
pub hash: [u8; 32],
pub size: u64,
pub paid: u64,
pub spliced: u64,
pub resumed: u64,
pub elapsed: Duration,
}
#[doc(hidden)]
pub trait FaultHints: Send + Sync {
fn annotate(&self, err: anyhow::Error, at: &FaultContext) -> anyhow::Error;
}
#[derive(Debug)]
#[non_exhaustive]
pub struct BundleOutcome {
pub report: BundleReport,
pub end: BundleEnd,
pub replaced: Option<PoolReplaced>,
}
#[derive(Debug)]
#[non_exhaustive]
pub enum BundleEnd {
Complete,
Interrupted,
FundingNeeded {
needed: FundingNeeded,
error: anyhow::Error,
},
Stopped(anyhow::Error),
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct BundleReport {
pub fetched: u64,
pub linked: u64,
pub skipped: u64,
pub failed: u64,
pub reused: u64,
pub reused_bytes: u64,
pub downloaded: u64,
pub paid_bytes: u64,
pub reconstructed: u64,
pub resumed_bytes: u64,
pub spliced_bytes: u64,
pub hints_ignored: u64,
pub failures: Vec<EntryFailure>,
pub warnings: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct EntryFailure {
pub path: String,
pub error: String,
}
pub struct BundlePuller {
ctx: engine::PullCtx<DynProvider>,
}
impl std::fmt::Debug for BundlePuller {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BundlePuller")
.field("chain", &self.ctx.chain)
.field("self_address", &self.ctx.self_address)
.finish_non_exhaustive()
}
}
#[must_use]
pub struct BundlePullerBuilder {
chain: ChainConfig,
site: PullSite,
endpoint: Endpoint,
store: Arc<dyn BuyerPoolStore>,
funding: Funding,
discovery: DiscoveryOptions,
nodes: NodeSelection,
relays: Vec<RelayUrl>,
namespace: Option<u64>,
jobs: usize,
max_lane_streams: usize,
max_rate_per_mb: u64,
max_blob_bytes: u64,
interactive: bool,
give_up_after: Option<Duration>,
observer: Option<Arc<dyn BundleObserver>>,
hints: Option<Arc<dyn FaultHints>>,
opens: UnconfirmedOpens,
}
impl std::fmt::Debug for BundlePullerBuilder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BundlePullerBuilder")
.field("chain", &self.chain)
.field("jobs", &self.jobs)
.finish_non_exhaustive()
}
}
impl BundlePuller {
pub fn builder(
chain: ChainConfig,
site: PullSite,
endpoint: Endpoint,
store: Arc<dyn BuyerPoolStore>,
funding: Funding,
discovery: DiscoveryOptions,
nodes: NodeSelection,
) -> BundlePullerBuilder {
BundlePullerBuilder {
chain,
site,
endpoint,
store,
funding,
discovery,
nodes,
relays: Vec::new(),
namespace: None,
jobs: 4,
max_lane_streams: 4,
max_rate_per_mb: 0,
max_blob_bytes: 1 << 40,
interactive: false,
give_up_after: None,
observer: None,
hints: None,
opens: UnconfirmedOpens::default(),
}
}
pub async fn fetch_manifest(
&self,
hash: [u8; 32],
out_dir: &Path,
use_cache: bool,
) -> anyhow::Result<BundleManifest> {
Box::pin(self.ctx.fetch_manifest(hash, out_dir, use_cache)).await
}
pub async fn pull(
&mut self,
manifest: &BundleManifest,
out_dir: &Path,
overwrite: bool,
cancel: &CancellationToken,
) -> BundleOutcome {
Box::pin(self.ctx.run(&manifest.entries, out_dir, overwrite, cancel)).await
}
#[must_use]
pub fn funding_shortfall(&self) -> Option<String> {
self.ctx.funding.shortfall()
}
pub async fn finish(self) {
self.ctx.writes.settle().await;
}
}
impl BundlePullerBuilder {
pub fn relays(mut self, relays: Vec<RelayUrl>) -> Self {
self.relays = relays;
self
}
pub const fn namespace(mut self, namespace: Option<u64>) -> Self {
self.namespace = namespace;
self
}
pub const fn jobs(mut self, jobs: usize) -> Self {
self.jobs = jobs;
self
}
pub const fn max_lane_streams(mut self, n: usize) -> Self {
self.max_lane_streams = n;
self
}
pub const fn max_rate_per_mb(mut self, rate: u64) -> Self {
self.max_rate_per_mb = rate;
self
}
pub const fn max_blob_bytes(mut self, bytes: u64) -> Self {
self.max_blob_bytes = bytes;
self
}
pub const fn stop(mut self, interactive: bool, give_up_after: Option<Duration>) -> Self {
self.interactive = interactive;
self.give_up_after = give_up_after;
self
}
pub fn observer(mut self, observer: Arc<dyn BundleObserver>) -> Self {
self.observer = Some(observer);
self
}
pub fn unconfirmed_opens(mut self, opens: UnconfirmedOpens) -> Self {
self.opens = opens;
self
}
#[doc(hidden)]
pub fn fault_hints(mut self, hints: Arc<dyn FaultHints>) -> Self {
self.hints = Some(hints);
self
}
pub async fn build(self) -> anyhow::Result<BundlePuller> {
let (signer, slot, deposit) = match self.funding {
Funding::SelfOwned { signer, deposit } => (signer, None, deposit),
Funding::Delegated { slot } => {
let deposit = DepositPolicy {
working_deposit: U256::ZERO,
max_approve: false,
adoption: ChainAdoption::Refused,
};
(slot.current().signer, Some(slot), deposit)
}
};
let rpc = crate::provider::build_provider(&self.chain.rpc_url, &signer)?.erased();
let contract = PaymentPool::new(self.chain.payment_pool, rpc.clone());
let token = contract
.usdc()
.call()
.await
.map_err(|e| anyhow::Error::new(e).context("read PaymentPool.usdc()"))?;
let ctx = engine::PullCtx {
voucher_dom: voucher_domain(self.chain.chain_id, self.chain.payment_pool),
slash_dom: slash_judge_domain(self.chain.chain_id, self.chain.slash_judge),
connections: Connections::new(self.endpoint.clone()),
self_address: signer.address(),
endpoint: self.endpoint,
store: self.store,
contract,
rpc,
signer,
token,
chain: self.chain,
site: self.site,
relays: self.relays,
nodes: self.nodes,
discovery: self.discovery,
max_rate_per_mb: self.max_rate_per_mb,
max_blob_bytes: self.max_blob_bytes,
deposit,
slot,
namespace_id: namespace_id(self.namespace),
ledgers: LaneLedgers::new(),
funding: RunFunding::reporting_opens_to(self.opens),
recovery: Arc::new(RecoveryGate::new()),
dedup_stats: engine::DedupStats::default(),
open_lock: tokio::sync::Mutex::new(()),
gate: engine::JobGate::new(self.jobs.max(1)),
lane_cap: LaneStreamCap::new(self.max_lane_streams),
health: Arc::new(PeerHealth::default()),
writes: OrderedWrites::default(),
stop: StopPolicy::new(
self.interactive,
self.give_up_after,
Arc::new(ProgressClock::new()),
),
demand: Arc::new(RunDemand::default()),
observer: self.observer,
hints: self.hints,
};
Ok(BundlePuller { ctx })
}
}
fn namespace_id(namespace: Option<u64>) -> [u8; 32] {
namespace.map_or(decdn_protocol::client::NO_NAMESPACE, |n| {
U256::from(n).to_be_bytes()
})
}