halldyll-core 0.1.0

Core scraping engine for Halldyll - high-performance async web scraper for AI agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12

//! Storage - Storage of snapshots and normalized documents

pub mod normalized;
pub mod snapshot;
pub mod warc;

pub use normalized::NormalizedStore;
pub use snapshot::{RawSnapshot, SnapshotStore};
pub use warc::{
    RotatingWarcWriter, WarcFileHandle, WarcInfo, WarcMetadata, WarcRequest, WarcResponse,
    WarcWriter,
};