warcat 0.3.0

Command-line tool and library for handling Web ARChive (WARC) files
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

use std::path::PathBuf;

use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, Deserialize, Serialize)]
pub enum WarcMessage {
    Metadata(Metadata),
    Header(Header),
    BlockChunk(BlockChunk),
    BlockEnd(BlockEnd),
    ExtractMetadata(ExtractMetadata),
    ExtractChunk(ExtractChunk),
    ExtractEnd(ExtractEnd),
    EndOfFile(EndOfFile),
}

#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Metadata {
    pub file: PathBuf,
    pub position: u64,
}

#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Header {
    pub version: String,
    pub fields: Vec<(String, String)>,
}

#[serde_with::serde_as]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct BlockChunk {
    #[serde_as(as = "serde_with::IfIsHumanReadable<serde_with::base64::Base64,serde_with::Bytes>")]
    pub data: Vec<u8>,
}

#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct BlockEnd {
    pub crc32: Option<u32>,
    pub crc32c: Option<u32>,
    pub xxh3: Option<u64>,
}

#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ExtractMetadata {
    pub has_content: bool,
    pub file_path_components: Vec<String>,
    pub is_truncated: bool,
}

#[serde_with::serde_as]
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ExtractChunk {
    #[serde_as(as = "serde_with::IfIsHumanReadable<serde_with::base64::Base64,serde_with::Bytes>")]
    pub data: Vec<u8>,
}

#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ExtractEnd {
    pub crc32: Option<u32>,
    pub crc32c: Option<u32>,
    pub xxh3: Option<u64>,
}

#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct EndOfFile {}