shrike 0.1.1

AT Protocol library for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

//! AT Protocol concurrent repo backfill engine.
//!
//! # Overview
//!
//! [`BackfillEngine`] downloads all repositories from a relay or PDS
//! concurrently.
//!
//! Key features:
//! - Cursor-based pagination with crash-recovery via the [`Checkpoint`] trait.
//! - Fisher-Yates batch shuffling to distribute load across PDS hosts.
//! - Configurable worker concurrency and batch size.
//! - Graceful shutdown via [`tokio_util::sync::CancellationToken`].

pub mod checkpoint;
pub mod engine;

pub use checkpoint::{Checkpoint, NoopCheckpoint};
pub use engine::{BackfillConfig, BackfillEngine, BackfillStats, shuffle_batch};

use thiserror::Error;

/// Errors produced by the backfill engine.
#[derive(Debug, Error)]
pub enum BackfillError {
    #[error("checkpoint error: {0}")]
    Checkpoint(String),
    #[error("sync error: {0}")]
    Sync(String),
    #[error("XRPC error: {0}")]
    Xrpc(#[from] crate::xrpc::Error),
}

#[cfg(test)]
#[allow(
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::panic,
    clippy::unreachable
)]
mod tests {
    use crate::backfill::*;

    #[test]
    fn noop_checkpoint() {
        let rt = tokio::runtime::Runtime::new().unwrap();
        let cp = NoopCheckpoint;
        rt.block_on(async {
            assert!(cp.load().await.unwrap().is_none());
            cp.save("cursor-123").await.unwrap();
            assert!(cp.load().await.unwrap().is_none()); // noop doesn't persist
        });
    }
}