ccdown 0.7.0

A polite and user-friendly downloader for Common Crawl data.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

use ccdown::download::{fetch_record, RecordOptions};
use flate2::write::GzEncoder;
use flate2::Compression;
use std::io::Write;
use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};

/// Build one .warc.gz member containing a WARC response record with `body`.
fn warc_gz_member(body: &[u8]) -> Vec<u8> {
    let http = format!(
        "HTTP/1.1 200 OK\r\nContent-Type: application/pdf\r\nContent-Length: {}\r\n\r\n",
        body.len()
    );
    let block_len = http.len() + body.len();
    let warc = format!(
        "WARC/1.0\r\nWARC-Type: response\r\nWARC-Target-URI: http://example.com/a.pdf\r\nContent-Length: {block_len}\r\n\r\n"
    );
    let mut record = Vec::new();
    record.extend_from_slice(warc.as_bytes());
    record.extend_from_slice(http.as_bytes());
    record.extend_from_slice(body);
    record.extend_from_slice(b"\r\n\r\n");
    let mut enc = GzEncoder::new(Vec::new(), Compression::default());
    enc.write_all(&record).unwrap();
    enc.finish().unwrap()
}

fn opts(server_url: &str) -> RecordOptions {
    RecordOptions {
        max_retries: 0,
        base_url: Some(format!("{server_url}/")),
        ..Default::default()
    }
}

#[tokio::test]
async fn fetches_pdf_record_from_warc() {
    let pdf = b"%PDF-1.4 fake pdf bytes";
    // two members: the target record followed by a decoy — fetch must stop after one
    let mut payload = warc_gz_member(pdf);
    payload.extend_from_slice(&warc_gz_member(b"%PDF decoy"));

    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/crawl-data/test.warc.gz"))
        .and(header("Range", "bytes=0-"))
        .respond_with(ResponseTemplate::new(206).set_body_bytes(payload))
        .mount(&server)
        .await;

    let record = fetch_record("crawl-data/test.warc.gz", 0, &opts(&server.uri()))
        .await
        .unwrap();
    assert_eq!(record.body, pdf);
    assert_eq!(record.warc_headers.get("warc-type").unwrap(), "response");
    assert_eq!(
        record.http_headers.get("content-type").unwrap(),
        "application/pdf"
    );
}

#[tokio::test]
async fn truncated_member_is_an_error() {
    let full = warc_gz_member(b"%PDF-1.4 something");
    let truncated = full[..full.len() / 2].to_vec();
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/t.warc.gz"))
        .respond_with(ResponseTemplate::new(206).set_body_bytes(truncated))
        .mount(&server)
        .await;
    let err = fetch_record("t.warc.gz", 0, &opts(&server.uri())).await;
    assert!(err.is_err());
}

#[tokio::test]
async fn record_over_max_bytes_is_an_error() {
    let body = vec![b'x'; 64 * 1024];
    let payload = warc_gz_member(&body);
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/big.warc.gz"))
        .respond_with(ResponseTemplate::new(206).set_body_bytes(payload))
        .mount(&server)
        .await;
    let options = RecordOptions {
        max_bytes: 1024,
        ..opts(&server.uri())
    };
    assert!(fetch_record("big.warc.gz", 0, &options).await.is_err());
}

#[tokio::test]
async fn unrecoverable_status_is_reported() {
    let server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path("/gone.warc.gz"))
        .respond_with(ResponseTemplate::new(404))
        .mount(&server)
        .await;
    let err = fetch_record("gone.warc.gz", 0, &opts(&server.uri()))
        .await
        .unwrap_err();
    assert!(err.is_unrecoverable());
}

#[tokio::test]
async fn sends_range_header_for_nonzero_offset() {
    let body = b"%PDF-1.4 at offset";
    let payload = warc_gz_member(body);

    let server = MockServer::start().await;
    // The mock only matches requests that carry the exact Range header.
    // An unmatched request returns 404, which makes fetch_record fail —
    // so the header match is verified implicitly by the successful return.
    Mock::given(method("GET"))
        .and(path("/off.warc.gz"))
        .and(header("Range", "bytes=12345-"))
        .respond_with(ResponseTemplate::new(206).set_body_bytes(payload))
        .mount(&server)
        .await;

    let record = fetch_record("off.warc.gz", 12345, &opts(&server.uri()))
        .await
        .unwrap();
    assert_eq!(record.body, body);
}