Struct WarcWriter 

pub struct WarcWriter<W> { /* private fields */ }

A writer which writes records to an output stream.

Implementations

impl<W: Write> WarcWriter<W>

pub fn new(w: W) -> Self

Create a new writer.

pub fn write(&mut self, record: &Record<BufferedBody>) -> Result<usize>

Write a single record.

The number of bytes written is returned upon success.

Examples found in repository

examples/write_file.rs (line 20)

fn main() -> Result<(), std::io::Error> {
    let date = Utc::now();
    let body = format!("wrote to the file on {}", date);
    let body = body.into_bytes();

    let mut headers = Record::<BufferedBody>::new();
    headers.set_warc_type(RecordType::WarcInfo);
    headers.set_date(date);
    headers
        .set_header(WarcHeader::IPAddress, "127.0.0.1")
        .expect("BUG: should be a valid IP address");
    let record = headers.add_body(body);

    let mut file = WarcWriter::from_path("warc_example.warc")?;

    let bytes_written = file.write(&record)?;

    println!("{} bytes written.", bytes_written);

    Ok(())
}

pub fn write_raw<B>( &mut self, headers: RawRecordHeader, body: &B, ) -> Result<usize>

where B: AsRef<[u8]>,

Write a single raw record.

The number of bytes written is returned upon success.

Examples found in repository

examples/write_raw.rs (line 35)

fn main() -> Result<(), std::io::Error> {
    let date = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
    let body = format!("wrote to the file on {}", date);
    let body = body.into_bytes();

    let headers = RawRecordHeader {
        version: "1.0".to_owned(),
        headers: vec![
            (
                WarcHeader::RecordID,
                Record::<BufferedBody>::generate_record_id().into_bytes(),
            ),
            (
                WarcHeader::WarcType,
                RecordType::WarcInfo.to_string().into_bytes(),
            ),
            (WarcHeader::Date, date.into_bytes()),
            (WarcHeader::IPAddress, "127.0.0.1".to_owned().into_bytes()),
            (
                WarcHeader::ContentLength,
                body.len().to_string().into_bytes(),
            ),
        ]
        .into_iter()
        .collect(),
    };

    let mut file = WarcWriter::from_path("warc_example.warc")?;

    let bytes_written = file.write_raw(headers, &body)?;

    println!("{} bytes written.", bytes_written);

    Ok(())
}

examples/write_gzip.rs (line 35)

fn main() -> Result<(), std::io::Error> {
    let date = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
    let body = format!("wrote to the file on {}", date);
    let body = body.into_bytes();

    let headers = RawRecordHeader {
        version: "1.0".to_owned(),
        headers: vec![
            (
                WarcHeader::RecordID,
                Record::<BufferedBody>::generate_record_id().into_bytes(),
            ),
            (
                WarcHeader::WarcType,
                RecordType::WarcInfo.to_string().into_bytes(),
            ),
            (WarcHeader::Date, date.into_bytes()),
            (WarcHeader::IPAddress, "127.0.0.1".to_owned().into_bytes()),
            (
                WarcHeader::ContentLength,
                body.len().to_string().into_bytes(),
            ),
        ]
        .into_iter()
        .collect(),
    };

    let mut file = WarcWriter::from_path_gzip("warc_example.warc.gz")?;

    let bytes_written = file.write_raw(headers, &body)?;

    // NB: the compression stream must be finish()ed, or the file will be truncated
    let gzip_stream = file.into_inner()?;
    gzip_stream.finish().into_result()?;

    println!("{} bytes written.", bytes_written);

    Ok(())
}

impl<W: Write> WarcWriter<BufWriter<W>>

pub fn into_inner(self) -> Result<W, IntoInnerError<BufWriter<W>>>

Consume this writer and return the inner writer.

Flushing Compressed Data Streams

This method is necessary to be called at the end of a GZIP-compressed stream. An extra call is needed to flush the buffer of data, and write a trailer to the output stream.

let gzip_stream = writer.into_inner()?;
gzip_writer.finish().into_result()?;

Examples found in repository

examples/write_gzip.rs (line 38)

fn main() -> Result<(), std::io::Error> {
    let date = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
    let body = format!("wrote to the file on {}", date);
    let body = body.into_bytes();

    let headers = RawRecordHeader {
        version: "1.0".to_owned(),
        headers: vec![
            (
                WarcHeader::RecordID,
                Record::<BufferedBody>::generate_record_id().into_bytes(),
            ),
            (
                WarcHeader::WarcType,
                RecordType::WarcInfo.to_string().into_bytes(),
            ),
            (WarcHeader::Date, date.into_bytes()),
            (WarcHeader::IPAddress, "127.0.0.1".to_owned().into_bytes()),
            (
                WarcHeader::ContentLength,
                body.len().to_string().into_bytes(),
            ),
        ]
        .into_iter()
        .collect(),
    };

    let mut file = WarcWriter::from_path_gzip("warc_example.warc.gz")?;

    let bytes_written = file.write_raw(headers, &body)?;

    // NB: the compression stream must be finish()ed, or the file will be truncated
    let gzip_stream = file.into_inner()?;
    gzip_stream.finish().into_result()?;

    println!("{} bytes written.", bytes_written);

    Ok(())
}

impl WarcWriter<BufWriter<File>>

pub fn from_path<P: AsRef<Path>>(path: P) -> Result<Self>

Create a new writer which writes to a file.

Examples found in repository

examples/write_file.rs (line 18)

fn main() -> Result<(), std::io::Error> {
    let date = Utc::now();
    let body = format!("wrote to the file on {}", date);
    let body = body.into_bytes();

    let mut headers = Record::<BufferedBody>::new();
    headers.set_warc_type(RecordType::WarcInfo);
    headers.set_date(date);
    headers
        .set_header(WarcHeader::IPAddress, "127.0.0.1")
        .expect("BUG: should be a valid IP address");
    let record = headers.add_body(body);

    let mut file = WarcWriter::from_path("warc_example.warc")?;

    let bytes_written = file.write(&record)?;

    println!("{} bytes written.", bytes_written);

    Ok(())
}

examples/write_raw.rs (line 33)

fn main() -> Result<(), std::io::Error> {
    let date = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
    let body = format!("wrote to the file on {}", date);
    let body = body.into_bytes();

    let headers = RawRecordHeader {
        version: "1.0".to_owned(),
        headers: vec![
            (
                WarcHeader::RecordID,
                Record::<BufferedBody>::generate_record_id().into_bytes(),
            ),
            (
                WarcHeader::WarcType,
                RecordType::WarcInfo.to_string().into_bytes(),
            ),
            (WarcHeader::Date, date.into_bytes()),
            (WarcHeader::IPAddress, "127.0.0.1".to_owned().into_bytes()),
            (
                WarcHeader::ContentLength,
                body.len().to_string().into_bytes(),
            ),
        ]
        .into_iter()
        .collect(),
    };

    let mut file = WarcWriter::from_path("warc_example.warc")?;

    let bytes_written = file.write_raw(headers, &body)?;

    println!("{} bytes written.", bytes_written);

    Ok(())
}

impl WarcWriter<BufWriter<Encoder<File>>>

pub fn from_path_gzip<P: AsRef<Path>>(path: P) -> Result<Self>

Create a new writer which writes to a GZIP-compressed file.

Examples found in repository

examples/write_gzip.rs (line 33)

fn main() -> Result<(), std::io::Error> {
    let date = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
    let body = format!("wrote to the file on {}", date);
    let body = body.into_bytes();

    let headers = RawRecordHeader {
        version: "1.0".to_owned(),
        headers: vec![
            (
                WarcHeader::RecordID,
                Record::<BufferedBody>::generate_record_id().into_bytes(),
            ),
            (
                WarcHeader::WarcType,
                RecordType::WarcInfo.to_string().into_bytes(),
            ),
            (WarcHeader::Date, date.into_bytes()),
            (WarcHeader::IPAddress, "127.0.0.1".to_owned().into_bytes()),
            (
                WarcHeader::ContentLength,
                body.len().to_string().into_bytes(),
            ),
        ]
        .into_iter()
        .collect(),
    };

    let mut file = WarcWriter::from_path_gzip("warc_example.warc.gz")?;

    let bytes_written = file.write_raw(headers, &body)?;

    // NB: the compression stream must be finish()ed, or the file will be truncated
    let gzip_stream = file.into_inner()?;
    gzip_stream.finish().into_result()?;

    println!("{} bytes written.", bytes_written);

    Ok(())
}