Struct ResumeCheckpoint

Source

pub struct ResumeCheckpoint {
    pub snapshot_id: String,
    pub chunk_id: String,
    pub line_offset: u64,
    pub articles_processed: u64,
    pub timestamp: DateTime<Utc>,
}

Expand description

Resume checkpoint for crash recovery.

Stores the current position in a snapshot download, allowing processing to be resumed after interruptions.

§Example

use wme_stream::ResumeCheckpoint;

let checkpoint = ResumeCheckpoint::new(
    "enwiki_namespace_0",
    "chunk_0",
    5000,
    1000,
);

assert_eq!(checkpoint.snapshot_id, "enwiki_namespace_0");
assert_eq!(checkpoint.line_offset, 5000);

Fields§

§snapshot_id: String

Snapshot identifier (e.g., “enwiki_namespace_0”)

§chunk_id: String

Chunk identifier (e.g., “chunk_0”)

§line_offset: u64

Line offset within chunk (NDJSON line number)

§articles_processed: u64

Number of articles processed so far

§timestamp: DateTime<Utc>

Checkpoint timestamp

Implementations§

Source §

impl ResumeCheckpoint

Source

pub fn new( snapshot_id: impl Into<String>, chunk_id: impl Into<String>, line_offset: u64, articles_processed: u64, ) -> Self

Create a new checkpoint with current timestamp.

§Arguments

snapshot_id - The snapshot being processed
chunk_id - The current chunk
line_offset - Current line position in NDJSON file
articles_processed - Total articles processed

§Example

use wme_stream::ResumeCheckpoint;

let checkpoint = ResumeCheckpoint::new(
    "enwiki_namespace_0",
    "chunk_0",
    5000,
    1000,
);

Source

pub async fn save(&self, path: impl Into<PathBuf>) -> Result<(), StreamError>

Save checkpoint to disk as JSON.

The file is named {snapshot_id}.checkpoint.json and stored in the specified directory.

§Arguments

path - Directory to save checkpoint (or full file path)

§Errors

Returns StreamError::Io if file write fails.

§Example

use wme_stream::ResumeCheckpoint;

let checkpoint = ResumeCheckpoint::new("enwiki", "chunk_0", 100, 50);
checkpoint.save("/data/checkpoints/").await?;

Source

pub async fn load(path: impl Into<PathBuf>) -> Result<Self, StreamError>

Load checkpoint from disk.

§Arguments

path - Path to checkpoint JSON file

§Errors

Returns StreamError::Resume if:

File not found
Invalid JSON
Missing fields

§Example

use wme_stream::ResumeCheckpoint;

let checkpoint = ResumeCheckpoint::load(
    "/data/checkpoints/enwiki.checkpoint.json"
).await?;

println!("Resuming from line {}", checkpoint.line_offset);

Source

pub fn checkpoint_path( base_dir: impl Into<PathBuf>, snapshot_id: &str, ) -> PathBuf

Get checkpoint file path for a snapshot.

Generates a standard checkpoint filename in the specified directory.

§Arguments

base_dir - Directory for checkpoint files
snapshot_id - Snapshot identifier

§Returns

Path like {base_dir}/{snapshot_id}.checkpoint.json

§Example

use wme_stream::ResumeCheckpoint;
use std::path::PathBuf;

let path = ResumeCheckpoint::checkpoint_path("/data/checkpoints", "enwiki");
assert_eq!(path, PathBuf::from("/data/checkpoints/enwiki.checkpoint.json"));