ptox 0.3.0

A language-neutral SDK for building file-search clients across many networks.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
//! The **retrieval half** of ptox: turn a [`crate::types::Address`] into bytes on disk.
//!
//! The search fabric emits `Address`es; this fabric consumes them. Fetchers are
//! registered per scheme, so retrieval is pluggable exactly like search.
//!
//! ```no_run
//! # async fn run() -> Result<(), ptox::FetchError> {
//! use ptox::Address;
//! let registry = ptox::FetchRegistry::with_defaults();
//! let address = Address::new("https", "https://example.org/file.bin");
//! let fetched = registry.fetch(&address, "file.bin", None).await?;
//! println!("{} bytes -> {}", fetched.bytes, fetched.path.display());
//! # Ok(())
//! # }
//! ```

#![forbid(unsafe_code)]

mod command;
#[cfg(feature = "file")]
mod file;
mod http;
#[cfg(feature = "ipfs")]
mod ipfs;

pub use command::CommandFetcher;
#[cfg(feature = "file")]
pub use file::FileFetcher;
pub use http::{HttpFetcher, RetryPolicy};
#[cfg(feature = "ipfs")]
pub use ipfs::{IpfsGatewayFetcher, DEFAULT_IPFS_GATEWAY};

use crate::merge::MergedResult;
use crate::types::Address;
use async_trait::async_trait;
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use thiserror::Error;

/// Errors that can occur while retrieving an address.
#[derive(Debug, Error)]
pub enum FetchError {
    /// No fetcher is registered for the address scheme.
    #[error("unsupported address scheme `{0}`")]
    Unsupported(String),
    /// Filesystem error.
    #[error("io error: {0}")]
    Io(#[from] std::io::Error),
    /// Network or protocol error.
    #[error("network error: {0}")]
    Network(String),
    /// A helper command exited non-zero.
    #[error("command failed: {0}")]
    Command(String),
    /// Retrieved bytes did not match the expected hash.
    #[error("hash mismatch: expected {expected}, got {actual}")]
    HashMismatch {
        /// The expected digest (lowercase hex).
        expected: String,
        /// The digest actually retrieved (lowercase hex).
        actual: String,
    },
    /// The result carried no address to fetch.
    #[error("result has no address")]
    NoAddress,
    /// The address was malformed for its scheme (e.g. an `ipfs://` URI with no
    /// content identifier).
    #[error("invalid address: {0}")]
    InvalidAddress(String),
    /// Every address of a result failed to retrieve.
    #[error("all {count} address(es) failed")]
    AllFailed {
        /// How many addresses were tried.
        count: usize,
        /// The individual failures, in the order the addresses were tried.
        errors: Vec<FetchError>,
    },
    /// A local-file fetch was refused by policy (outside every configured root).
    #[error("access denied: {0}")]
    AccessDenied(String),
}

/// A single retrieval job: one address, one destination path.
#[derive(Debug, Clone)]
pub struct FetchRequest {
    /// The address to retrieve.
    pub address: Address,
    /// Where the bytes should be written.
    pub destination: PathBuf,
    /// Expected sha256 (lowercase hex); verified after retrieval.
    pub expected_sha256: Option<String>,
}

/// The outcome of a successful retrieval.
#[derive(Debug, Clone)]
pub struct Fetched {
    /// Path the bytes were written to.
    pub path: PathBuf,
    /// Number of bytes written.
    pub bytes: u64,
    /// sha256 of the retrieved bytes (lowercase hex), when computed.
    pub sha256: Option<String>,
    /// The address the bytes were actually retrieved from.
    ///
    /// With multi-address failover ([`FetchRegistry::fetch_merged`]) this is the
    /// first address that succeeded, which may not have been the preferred one.
    pub address: Address,
}

/// A progress observer, called with the cumulative number of bytes retrieved so
/// far. It is invoked once per received chunk, on the retrieval task, so it must
/// be cheap and must not block.
pub type ProgressFn<'a> = dyn Fn(u64) + Send + Sync + 'a;

/// One way of turning an address into bytes (one protocol).
#[async_trait]
pub trait Fetcher: Send + Sync {
    /// A stable name for diagnostics.
    fn name(&self) -> &str;
    /// The address schemes this fetcher handles.
    fn schemes(&self) -> &[String];
    /// Retrieve `request`, writing the bytes to its destination.
    async fn fetch(&self, request: &FetchRequest) -> Result<Fetched, FetchError>;

    /// Like [`Fetcher::fetch`], but report the cumulative number of retrieved
    /// bytes to `progress` as they arrive.
    ///
    /// The default implementation ignores `progress` and calls
    /// [`Fetcher::fetch`], so a fetcher that cannot observe progress need not
    /// implement this method. A caller that needs progress should use this
    /// method; a caller that does not can keep calling [`Fetcher::fetch`].
    ///
    /// `HttpFetcher` reports each received chunk; `CommandFetcher` polls the
    /// destination file size while its command runs.
    async fn fetch_with_progress(
        &self,
        request: &FetchRequest,
        progress: &ProgressFn<'_>,
    ) -> Result<Fetched, FetchError> {
        let _ = progress;
        self.fetch(request).await
    }
}

/// Dispatches an `Address` to the registered fetcher for its scheme.
#[derive(Default)]
pub struct FetchRegistry {
    fetchers: Vec<Arc<dyn Fetcher>>,
}

impl FetchRegistry {
    /// An empty registry with no fetchers.
    pub fn new() -> Self {
        Self::default()
    }

    /// Registry with the built-in fetchers: `http`/`https`, an `ipfs` gateway
    /// fetcher when the `ipfs` feature is enabled, and a `file` fetcher when the
    /// `file` feature is enabled.
    ///
    /// The IPFS gateway is `DEFAULT_IPFS_GATEWAY` (an `IpfsGatewayFetcher`);
    /// build the registry yourself with `IpfsGatewayFetcher::new` to choose a
    /// different gateway. The `file` fetcher is built with `FileFetcher::new()`
    /// (any readable path); register a `FileFetcher::with_allowed_roots` yourself
    /// to restrict it.
    pub fn with_defaults() -> Self {
        let mut registry = Self::new();
        registry.register(Arc::new(HttpFetcher::new()));
        #[cfg(feature = "ipfs")]
        registry.register(Arc::new(IpfsGatewayFetcher::with_default_gateway()));
        #[cfg(feature = "file")]
        registry.register(Arc::new(FileFetcher::new()));
        registry
    }

    /// Register `fetcher`; its schemes become resolvable.
    pub fn register(&mut self, fetcher: Arc<dyn Fetcher>) -> &mut Self {
        self.fetchers.push(fetcher);
        self
    }

    /// The first registered fetcher that handles `scheme`, if any.
    pub fn fetcher_for(&self, scheme: &str) -> Option<Arc<dyn Fetcher>> {
        self.fetchers
            .iter()
            .find(|fetcher| fetcher.schemes().iter().any(|name| name == scheme))
            .cloned()
    }

    /// Every address scheme this registry can resolve, sorted and deduplicated.
    ///
    /// This is the authoritative answer to "which schemes work?"; a client no
    /// longer has to infer it from its own config. A scheme registered by more
    /// than one fetcher appears once — the **first** registered fetcher still
    /// wins at fetch time (see [`FetchRegistry::fetcher_for`]). For the
    /// per-fetcher breakdown, including registration precedence, print the
    /// registry (it implements [`std::fmt::Display`]).
    pub fn schemes(&self) -> Vec<String> {
        let mut schemes: Vec<String> = self
            .fetchers
            .iter()
            .flat_map(|fetcher| fetcher.schemes().iter().cloned())
            .collect();
        schemes.sort();
        schemes.dedup();
        schemes
    }

    /// Fetch `address` into `destination`, verifying `expected_sha256` if given.
    pub async fn fetch(
        &self,
        address: &Address,
        destination: impl Into<PathBuf>,
        expected_sha256: Option<String>,
    ) -> Result<Fetched, FetchError> {
        self.fetch_with_progress(address, destination, expected_sha256, &|_: u64| {})
            .await
    }

    /// Like [`FetchRegistry::fetch`], but report cumulative retrieved bytes to
    /// `progress` as they arrive. See [`Fetcher::fetch_with_progress`].
    pub async fn fetch_with_progress(
        &self,
        address: &Address,
        destination: impl Into<PathBuf>,
        expected_sha256: Option<String>,
        progress: &ProgressFn<'_>,
    ) -> Result<Fetched, FetchError> {
        let fetcher = self
            .fetcher_for(&address.scheme)
            .ok_or_else(|| FetchError::Unsupported(address.scheme.clone()))?;
        fetcher
            .fetch_with_progress(
                &FetchRequest {
                    address: address.clone(),
                    destination: destination.into(),
                    expected_sha256,
                },
                progress,
            )
            .await
    }

    /// Fetch a merged result, trying its addresses in preference order until
    /// one succeeds (multi-source failover), and verifying its sha256.
    ///
    /// Addresses are ordered by scheme (`https` > `http` > `ipfs` > `magnet` >
    /// other), preserving the result's own order within a scheme; see
    /// [`preferred_addresses`]. The winning address is recorded on
    /// [`Fetched::address`]. Fails with [`FetchError::NoAddress`] when the
    /// result has no addresses, or [`FetchError::AllFailed`] (carrying every
    /// attempt's error) when all of them fail.
    pub async fn fetch_merged(
        &self,
        result: &MergedResult,
        destination: impl Into<PathBuf>,
    ) -> Result<Fetched, FetchError> {
        self.fetch_merged_with_progress(result, destination, &|_: u64| {})
            .await
    }

    /// Like [`FetchRegistry::fetch_merged`], but report cumulative retrieved
    /// bytes to `progress` as they arrive.
    ///
    /// The reported total is kept monotonic across failover: if one address
    /// fails part-way and the next succeeds, the bar never jumps backwards, and
    /// the final report equals the size of the address that won.
    pub async fn fetch_merged_with_progress(
        &self,
        result: &MergedResult,
        destination: impl Into<PathBuf>,
        progress: &ProgressFn<'_>,
    ) -> Result<Fetched, FetchError> {
        let destination = destination.into();
        let addresses = preferred_addresses(result);
        if addresses.is_empty() {
            return Err(FetchError::NoAddress);
        }

        // A failed attempt may already have reported bytes; clamp later reports
        // to the high-water mark so progress never regresses.
        let high_water = std::sync::atomic::AtomicU64::new(0);
        let mut errors = Vec::new();
        for address in addresses {
            let expected = expected_sha256(result, address);
            let monotonic = |bytes: u64| {
                let previous = high_water.fetch_max(bytes, std::sync::atomic::Ordering::Relaxed);
                if bytes > previous {
                    progress(bytes);
                }
            };
            match self
                .fetch_with_progress(address, destination.clone(), expected, &monotonic)
                .await
            {
                Ok(fetched) => return Ok(fetched),
                Err(error) => errors.push(error),
            }
        }
        let count = errors.len();
        Err(FetchError::AllFailed { count, errors })
    }
}

impl std::fmt::Display for FetchRegistry {
    /// A diagnostics summary of the registry, one line per registered fetcher in
    /// registration order:
    ///
    /// ```text
    /// http (http, https)
    /// ipfs-gateway (ipfs)
    /// ```
    ///
    /// The format is intended for humans and error messages, not for machines;
    /// the machine-readable list is [`FetchRegistry::schemes`].
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        if self.fetchers.is_empty() {
            return formatter.write_str("(no fetchers)");
        }
        for (index, fetcher) in self.fetchers.iter().enumerate() {
            if index > 0 {
                formatter.write_str("\n")?;
            }
            write!(formatter, "{} (", fetcher.name())?;
            for (scheme_index, scheme) in fetcher.schemes().iter().enumerate() {
                if scheme_index > 0 {
                    formatter.write_str(", ")?;
                }
                formatter.write_str(scheme)?;
            }
            formatter.write_str(")")?;
        }
        Ok(())
    }
}

/// sha256 of a byte slice, as lowercase hex.
pub fn sha256_hex(bytes: &[u8]) -> String {
    hex::encode(Sha256::digest(bytes))
}

/// sha256 of a file, as lowercase hex. Streams the file in chunks so a large
/// download is never read into memory at once.
pub async fn sha256_file(path: &Path) -> std::io::Result<String> {
    use tokio::io::AsyncReadExt;

    let mut file = tokio::fs::File::open(path).await?;
    let mut hasher = Sha256::new();
    let mut buffer = vec![0u8; 64 * 1024];
    loop {
        let read = file.read(&mut buffer).await?;
        if read == 0 {
            break;
        }
        hasher.update(&buffer[..read]);
    }
    Ok(hex::encode(hasher.finalize()))
}

/// Verify `actual` against an optional `expected` sha256 (case-insensitive).
pub fn verify_sha256(expected: Option<&str>, actual: &str) -> Result<(), FetchError> {
    match expected {
        Some(expected) if !expected.eq_ignore_ascii_case(actual) => Err(FetchError::HashMismatch {
            expected: expected.to_owned(),
            actual: actual.to_owned(),
        }),
        _ => Ok(()),
    }
}

/// Pick the preferred address of a result: `https` > `http` > `ipfs` > `magnet` > other.
///
/// The ordering is shared with [`crate::merge::suggest_filename`] so a suggested
/// filename and the address that is actually fetched agree.
pub fn preferred_address(result: &MergedResult) -> Option<&Address> {
    crate::merge::preferred_address(result)
}

/// Every address of a result, ordered by preference: `https` > `http` > `ipfs`
/// > `magnet` > other, preserving the result's own order within a scheme.
///
/// This is the order [`FetchRegistry::fetch_merged`] tries addresses in when it
/// fails over from a dead mirror.
pub fn preferred_addresses(result: &MergedResult) -> Vec<&Address> {
    let mut addresses: Vec<&Address> = result.addresses.iter().collect();
    addresses.sort_by_key(|address| crate::merge::address_rank(&address.scheme));
    addresses
}

/// The expected sha256 for a result/address, if either carries one.
pub fn expected_sha256(result: &MergedResult, address: &Address) -> Option<String> {
    result
        .hashes
        .get("sha256")
        .or_else(|| {
            address
                .hashes
                .as_ref()
                .and_then(|hashes| hashes.get("sha256"))
        })
        .cloned()
}

/// `<destination>.part` — written first, then renamed into place.
pub(crate) fn temp_path(destination: &Path) -> PathBuf {
    let mut name = destination
        .file_name()
        .map(|name| name.to_string_lossy().into_owned())
        .unwrap_or_else(|| "download".to_owned());
    name.push_str(".part");
    destination.with_file_name(name)
}