Struct WarcReader 

pub struct WarcReader<R> { /* private fields */ }

A reader which iteratively parses WARC records from a stream.

Implementations

impl<R: BufRead> WarcReader<R>

pub fn new(r: R) -> Self

Create a new reader.

pub fn iter_raw_records(self) -> RawRecordIter<R>

Create an iterator over all of the raw records read.

This only does well-formedness checks on the headers. See RawRecordHeader for more information.

Examples found in repository

examples/read_raw.rs (line 8)

fn main() -> Result<(), std::io::Error> {
    let file = WarcReader::from_path("warc_example.warc")?;

    let mut count = 0;
    for record in file.iter_raw_records() {
        count += 1;
        match record {
            Err(err) => println!("ERROR: {}\r\n", err),
            Ok((headers, _)) => {
                println!(
                    "{}: {}",
                    WarcHeader::RecordID,
                    String::from_utf8_lossy(headers.as_ref().get(&WarcHeader::RecordID).unwrap())
                );
                println!(
                    "{}: {}",
                    WarcHeader::Date,
                    String::from_utf8_lossy(headers.as_ref().get(&WarcHeader::Date).unwrap())
                );
                println!();
            }
        }
    }

    println!("Total records: {}", count);

    Ok(())
}

pub fn iter_records(self) -> RecordIter<R>

Create an iterator over all of the records read.

This will fully build each record and check it for semantic correctness. See the Record type for more information.

Examples found in repository

examples/read_file.rs (line 8)

fn main() -> Result<(), std::io::Error> {
    let file = WarcReader::from_path("warc_example.warc")?;

    let mut count = 0;
    for record in file.iter_records() {
        count += 1;
        match record {
            Err(err) => println!("ERROR: {}\r\n", err),
            Ok(record) => {
                println!("{}: {}", WarcHeader::RecordID, record.warc_id(),);
                println!("{}: {}", WarcHeader::Date, record.date(),);
                println!();
            }
        }
    }

    println!("Total records: {}", count);

    Ok(())
}

examples/read_gzip.rs (line 8)

fn main() -> Result<(), std::io::Error> {
    let file = WarcReader::from_path_gzip("warc_example.warc.gz")?;

    let mut count = 0;
    for record in file.iter_records() {
        count += 1;
        match record {
            Err(err) => println!("ERROR: {}\r\n", err),
            Ok(record) => {
                println!("{}: {}", WarcHeader::RecordID, record.warc_id());
                println!("{}: {}", WarcHeader::Date, record.date());
                println!();
            }
        }
    }

    println!("Total records: {}", count);

    Ok(())
}

pub fn stream_records(&mut self) -> StreamingIter<'_, R>

Create a streaming iterator over all of the records read.

This will build each record header, and allow the caller to decide whether to read the body or not.

Examples found in repository

examples/read_filtered.rs (line 26)

fn main() -> std::io::Result<()> {
    let mut args = std::env::args_os().skip(1);

    let warc_name = args
        .next()
        .ok_or_else(|| usage_err!("compressed warc filename not supplied"))?;

    let filtered_file_names: Vec<_> = args.map(|s| s.to_string_lossy().to_string()).collect();
    if filtered_file_names.is_empty() {
        Err(usage_err!("one or more filtered file names not supplied"))?;
    }

    let mut file = WarcReader::from_path_gzip(warc_name)?;

    let mut count = 0;
    let mut skipped = 0;
    let mut stream_iter = file.stream_records();
    while let Some(record) = stream_iter.next_item() {
        let record = record.expect("read of headers ok");
        count += 1;
        match record.header(WarcHeader::TargetURI).map(|s| s.to_string()) {
            Some(v) if has_matching_filename(&v, &filtered_file_names) => {
                println!("Matches filename, skipping record");
                skipped += 1;
            }
            _ => {
                let buffered = record.into_buffered().expect("read of record ok");
                println!(
                    "Found record. Data:\n{}",
                    String::from_utf8_lossy(buffered.body())
                );
            }
        }
    }

    println!("Total records: {}\nSkipped records: {}", count, skipped);

    Ok(())
}

impl WarcReader<BufReader<File>>

pub fn from_path<P: AsRef<Path>>(path: P) -> Result<Self>

Create a new reader which reads from file.

Examples found in repository

examples/read_file.rs (line 5)

fn main() -> Result<(), std::io::Error> {
    let file = WarcReader::from_path("warc_example.warc")?;

    let mut count = 0;
    for record in file.iter_records() {
        count += 1;
        match record {
            Err(err) => println!("ERROR: {}\r\n", err),
            Ok(record) => {
                println!("{}: {}", WarcHeader::RecordID, record.warc_id(),);
                println!("{}: {}", WarcHeader::Date, record.date(),);
                println!();
            }
        }
    }

    println!("Total records: {}", count);

    Ok(())
}

examples/read_raw.rs (line 5)

fn main() -> Result<(), std::io::Error> {
    let file = WarcReader::from_path("warc_example.warc")?;

    let mut count = 0;
    for record in file.iter_raw_records() {
        count += 1;
        match record {
            Err(err) => println!("ERROR: {}\r\n", err),
            Ok((headers, _)) => {
                println!(
                    "{}: {}",
                    WarcHeader::RecordID,
                    String::from_utf8_lossy(headers.as_ref().get(&WarcHeader::RecordID).unwrap())
                );
                println!(
                    "{}: {}",
                    WarcHeader::Date,
                    String::from_utf8_lossy(headers.as_ref().get(&WarcHeader::Date).unwrap())
                );
                println!();
            }
        }
    }

    println!("Total records: {}", count);

    Ok(())
}

impl WarcReader<BufReader<MultiDecoder<BufReader<File>>>>

pub fn from_path_gzip<P: AsRef<Path>>(path: P) -> Result<Self>

Create a new reader which reads from a compressed file.

Only GZIP compression is currently supported.

Examples found in repository

examples/read_gzip.rs (line 5)

fn main() -> Result<(), std::io::Error> {
    let file = WarcReader::from_path_gzip("warc_example.warc.gz")?;

    let mut count = 0;
    for record in file.iter_records() {
        count += 1;
        match record {
            Err(err) => println!("ERROR: {}\r\n", err),
            Ok(record) => {
                println!("{}: {}", WarcHeader::RecordID, record.warc_id());
                println!("{}: {}", WarcHeader::Date, record.date());
                println!();
            }
        }
    }

    println!("Total records: {}", count);

    Ok(())
}

examples/read_filtered.rs (line 22)

fn main() -> std::io::Result<()> {
    let mut args = std::env::args_os().skip(1);

    let warc_name = args
        .next()
        .ok_or_else(|| usage_err!("compressed warc filename not supplied"))?;

    let filtered_file_names: Vec<_> = args.map(|s| s.to_string_lossy().to_string()).collect();
    if filtered_file_names.is_empty() {
        Err(usage_err!("one or more filtered file names not supplied"))?;
    }

    let mut file = WarcReader::from_path_gzip(warc_name)?;

    let mut count = 0;
    let mut skipped = 0;
    let mut stream_iter = file.stream_records();
    while let Some(record) = stream_iter.next_item() {
        let record = record.expect("read of headers ok");
        count += 1;
        match record.header(WarcHeader::TargetURI).map(|s| s.to_string()) {
            Some(v) if has_matching_filename(&v, &filtered_file_names) => {
                println!("Matches filename, skipping record");
                skipped += 1;
            }
            _ => {
                let buffered = record.into_buffered().expect("read of record ok");
                println!(
                    "Found record. Data:\n{}",
                    String::from_utf8_lossy(buffered.body())
                );
            }
        }
    }

    println!("Total records: {}\nSkipped records: {}", count, skipped);

    Ok(())
}