document_svg/document/
warc.rs

1//! Bounded WARC 1.0/1.1 web-archive record previews.
2//!
3//! WARC records may contain arbitrary HTTP payloads and private metadata. The
4//! reader validates record framing and content lengths, then renders only
5//! record headers and a bounded HTTP status/MIME summary. Payloads are skipped
6//! and no archived URL or embedded resource is opened.
7
8use std::io::Read;
9use std::path::Path;
10
11use flate2::read::MultiGzDecoder;
12
13use crate::convert::{ConvertOptions, PageConsumer, read_limited_file};
14use crate::document::html::{HtmlBlock, render_blocks_to_pages};
15use crate::error::{Error, Result};
16use crate::table::{TableAlign, TableData};
17
18const MAX_WARC_BYTES: u64 = 128 * 1024 * 1024;
19const MAX_WARC_DECOMPRESSED_BYTES: u64 = 128 * 1024 * 1024;
20const MAX_WARC_RECORD_BYTES: u64 = 64 * 1024 * 1024;
21const MAX_WARC_RECORDS: usize = 200_000;
22const MAX_WARC_HEADER_BYTES: usize = 1024 * 1024;
23const MAX_WARC_STRING_BYTES: usize = 2 * 1024 * 1024;
24const MAX_WARC_TEXT_BYTES: usize = 128 * 1024 * 1024;
25
26pub(crate) fn looks_like_prefix(prefix: &[u8]) -> bool {
27    prefix
28        .split(|byte| *byte == b'\n')
29        .map(|line| String::from_utf8_lossy(line))
30        .map(|line| line.trim_end_matches('\r').trim().to_owned())
31        .any(|line| line.starts_with("WARC/1."))
32}
33
34pub(crate) fn looks_like_gzip_file(path: &Path) -> bool {
35    let Ok(file) = std::fs::File::open(path) else {
36        return false;
37    };
38    let mut decoder = MultiGzDecoder::new(file);
39    let mut prefix = Vec::new();
40    if Read::take(&mut decoder, 4096)
41        .read_to_end(&mut prefix)
42        .is_err()
43    {
44        return false;
45    }
46    looks_like_prefix(&prefix)
47}
48
49struct WarcPageSink<'a> {
50    inner: &'a mut dyn PageConsumer,
51    warnings: &'a [String],
52}
53
54impl PageConsumer for WarcPageSink<'_> {
55    fn consume(&mut self, mut page: crate::ir::Page) -> Result<()> {
56        page.source_format = "warc".into();
57        if page.title.is_empty() {
58            page.title = "WARC web archive".into();
59        }
60        page.description =
61            "WARC record metadata is rendered safely; archived payloads and resources are not opened".into();
62        for warning in self.warnings {
63            page.warn(warning.clone());
64        }
65        self.inner.consume(page)
66    }
67}
68
69pub(crate) fn convert(
70    path: &Path,
71    options: &ConvertOptions,
72    sink: &mut dyn PageConsumer,
73) -> Result<Vec<String>> {
74    let compressed = path
75        .file_name()
76        .and_then(|value| value.to_str())
77        .is_some_and(|name| name.to_ascii_lowercase().ends_with(".warc.gz"));
78    let bytes = if compressed {
79        let file = std::fs::File::open(path)?;
80        let mut decoder = MultiGzDecoder::new(file);
81        let mut data = Vec::new();
82        Read::take(
83            &mut decoder,
84            options
85                .max_input_bytes
86                .min(MAX_WARC_DECOMPRESSED_BYTES)
87                .saturating_add(1),
88        )
89        .read_to_end(&mut data)?;
90        if data.len() as u64 > options.max_input_bytes.min(MAX_WARC_DECOMPRESSED_BYTES) {
91            return Err(Error::LimitExceeded(format!(
92                "decompressed WARC input exceeds {} bytes",
93                options.max_input_bytes.min(MAX_WARC_DECOMPRESSED_BYTES)
94            )));
95        }
96        data
97    } else {
98        read_limited_file(
99            path,
100            options.max_input_bytes.min(MAX_WARC_BYTES),
101            "WARC input",
102        )?
103    };
104    let (table, metadata, warnings) = parse_warc(&bytes)?;
105    let blocks = vec![
106        HtmlBlock::Heading {
107            level: 1,
108            text: "WARC web archive".into(),
109        },
110        HtmlBlock::Paragraph { text: metadata },
111        HtmlBlock::Table(table),
112    ];
113    let mut page_sink = WarcPageSink {
114        inner: sink,
115        warnings: &warnings,
116    };
117    render_blocks_to_pages(&blocks, &mut page_sink, options)?;
118    Ok(warnings)
119}
120
121fn parse_warc(bytes: &[u8]) -> Result<(TableData, String, Vec<String>)> {
122    if bytes.len() as u64 > MAX_WARC_BYTES || bytes.len() > MAX_WARC_TEXT_BYTES {
123        return Err(Error::LimitExceeded(format!(
124            "WARC input exceeds {MAX_WARC_BYTES} bytes"
125        )));
126    }
127    let mut rows = Vec::new();
128    let mut position = 0usize;
129    let mut versions = Vec::new();
130    let mut payload_records = 0usize;
131    let mut target_mask_count = 0usize;
132    while position < bytes.len() {
133        while bytes
134            .get(position..)
135            .is_some_and(|tail| tail.starts_with(b"\r\n") || tail.starts_with(b"\n"))
136        {
137            position += if bytes[position] == b'\r' { 2 } else { 1 };
138        }
139        if position >= bytes.len() {
140            break;
141        }
142        if rows.len() >= MAX_WARC_RECORDS {
143            return Err(Error::LimitExceeded(format!(
144                "WARC record count exceeds {MAX_WARC_RECORDS}"
145            )));
146        }
147        let header_end = find_header_end(bytes, position).ok_or_else(|| {
148            Error::InvalidInput("WARC record header is missing its blank-line terminator".into())
149        })?;
150        if header_end.0.saturating_sub(position) > MAX_WARC_HEADER_BYTES {
151            return Err(Error::LimitExceeded(format!(
152                "WARC header exceeds {MAX_WARC_HEADER_BYTES} bytes"
153            )));
154        }
155        let header = std::str::from_utf8(&bytes[position..header_end.0])
156            .map_err(|error| Error::InvalidInput(format!("WARC header must be UTF-8: {error}")))?;
157        let mut lines = header.lines();
158        let version = lines
159            .next()
160            .map(str::trim)
161            .ok_or_else(|| Error::InvalidInput("WARC record header is empty".into()))?;
162        if !version.starts_with("WARC/1.") {
163            return Err(Error::InvalidInput(format!(
164                "unsupported WARC record version `{version}`"
165            )));
166        }
167        versions.push(version.to_owned());
168        let mut record_type = String::new();
169        let mut target = String::new();
170        let mut date = String::new();
171        let mut content_type = String::new();
172        let mut content_length = None::<u64>;
173        for line in lines {
174            let Some((name, value)) = line.split_once(':') else {
175                continue;
176            };
177            let value = value.trim();
178            match name.trim().to_ascii_lowercase().as_str() {
179                "warc-type" => record_type = value.to_owned(),
180                "warc-target-uri" => target = value.to_owned(),
181                "warc-date" => date = value.to_owned(),
182                "content-type" => content_type = value.to_owned(),
183                "content-length" => {
184                    content_length = Some(value.parse::<u64>().map_err(|_| {
185                        Error::InvalidInput(format!("invalid WARC Content-Length `{value}`"))
186                    })?);
187                }
188                _ => {}
189            }
190        }
191        let length = content_length
192            .ok_or_else(|| Error::InvalidInput("WARC record is missing Content-Length".into()))?;
193        if length > MAX_WARC_RECORD_BYTES {
194            return Err(Error::LimitExceeded(format!(
195                "WARC record payload exceeds {MAX_WARC_RECORD_BYTES} bytes"
196            )));
197        }
198        let body_start = header_end.0 + header_end.1;
199        let body_end = body_start
200            .checked_add(usize::try_from(length).map_err(|_| {
201                Error::LimitExceeded("WARC Content-Length does not fit in memory".into())
202            })?)
203            .ok_or_else(|| Error::LimitExceeded("WARC record offset overflowed".into()))?;
204        if body_end > bytes.len() {
205            return Err(Error::InvalidInput(format!(
206                "WARC record payload is truncated: need {length} bytes"
207            )));
208        }
209        let body = &bytes[body_start..body_end];
210        let (status, body_mime) = summarize_http_body(body);
211        let mime = if body_mime.is_empty() {
212            content_type.clone()
213        } else {
214            body_mime
215        };
216        let (safe_target, masked) = mask_url(&target);
217        target_mask_count += masked;
218        if !body.is_empty() {
219            payload_records += 1;
220        }
221        rows.push(vec![
222            if record_type.is_empty() {
223                "unknown".into()
224            } else {
225                record_type
226            },
227            safe_target,
228            status,
229            mime,
230            date,
231            length.to_string(),
232        ]);
233        position = body_end;
234    }
235    if rows.is_empty() {
236        return Err(Error::InvalidInput("WARC input contains no records".into()));
237    }
238    let mut warnings = vec![
239        "WARC target URLs and embedded payloads are displayed inertly; no archived URL, header, cookie, script, or body is opened or replayed".into(),
240    ];
241    if payload_records > 0 {
242        warnings.push(format!(
243            "{payload_records} WARC payload record(s) were skipped after length validation"
244        ));
245    }
246    if target_mask_count > 0 {
247        warnings.push(format!(
248            "{target_mask_count} WARC target URI query value(s) were masked"
249        ));
250    }
251    let distinct_versions = versions
252        .into_iter()
253        .collect::<std::collections::BTreeSet<_>>()
254        .into_iter()
255        .collect::<Vec<_>>()
256        .join(", ");
257    let metadata = format!(
258        "WARC version(s): {distinct_versions}\nRecords: {}",
259        rows.len()
260    );
261    Ok((
262        TableData {
263            headers: vec![
264                "Type".into(),
265                "Target URI".into(),
266                "HTTP status".into(),
267                "MIME".into(),
268                "Date".into(),
269                "Length".into(),
270            ],
271            rows,
272            alignments: vec![TableAlign::Left; 6],
273            raw_source: String::new(),
274        },
275        metadata,
276        warnings,
277    ))
278}
279
280fn find_header_end(bytes: &[u8], start: usize) -> Option<(usize, usize)> {
281    let tail = bytes.get(start..)?;
282    let crlf = tail.windows(4).position(|window| window == b"\r\n\r\n");
283    let lf = tail.windows(2).position(|window| window == b"\n\n");
284    match (crlf, lf) {
285        (Some(a), Some(b)) if a <= b => Some((start + a, 4)),
286        (Some(a), _) => Some((start + a, 4)),
287        (_, Some(b)) => Some((start + b, 2)),
288        _ => None,
289    }
290}
291
292fn summarize_http_body(body: &[u8]) -> (String, String) {
293    let prefix = &body[..body.len().min(64 * 1024)];
294    let text = String::from_utf8_lossy(prefix);
295    let mut lines = text.lines();
296    let status = lines
297        .next()
298        .and_then(|line| line.split_whitespace().nth(1))
299        .filter(|value| value.len() == 3 && value.bytes().all(|byte| byte.is_ascii_digit()))
300        .unwrap_or("-")
301        .to_owned();
302    let mut mime = String::new();
303    for line in lines {
304        if let Some((name, value)) = line.split_once(':')
305            && name.trim().eq_ignore_ascii_case("content-type")
306        {
307            mime = value.trim().to_owned();
308            break;
309        }
310        if line.trim().is_empty() {
311            break;
312        }
313    }
314    (status, mime)
315}
316
317fn mask_url(url: &str) -> (String, usize) {
318    let Some((prefix, query)) = url.split_once('?') else {
319        return (truncate(url), 0);
320    };
321    let mut count = 0usize;
322    let query = query
323        .split('&')
324        .map(|part| {
325            let Some((key, value)) = part.split_once('=') else {
326                return part.to_owned();
327            };
328            let normalized = key.to_ascii_lowercase().replace(['-', '_'], "");
329            if [
330                "token",
331                "secret",
332                "password",
333                "apikey",
334                "authorization",
335                "cookie",
336                "session",
337                "credential",
338            ]
339            .iter()
340            .any(|needle| normalized.contains(needle))
341            {
342                count += 1;
343                format!("{key}=***")
344            } else {
345                format!("{key}={value}")
346            }
347        })
348        .collect::<Vec<_>>()
349        .join("&");
350    (truncate(&format!("{prefix}?{query}")), count)
351}
352
353fn truncate(value: &str) -> String {
354    if value.len() <= MAX_WARC_STRING_BYTES {
355        return value.to_owned();
356    }
357    let mut end = MAX_WARC_STRING_BYTES;
358    while !value.is_char_boundary(end) {
359        end -= 1;
360    }
361    format!("{}…", &value[..end])
362}
363
364#[cfg(test)]
365mod tests {
366    use super::*;
367
368    #[test]
369    fn previews_warc_records_and_skips_payloads() {
370        let source = b"WARC/1.1\r\nWARC-Type: response\r\nWARC-Target-URI: https://example.invalid/x?token=secret&ok=1\r\nWARC-Date: 2026-09-16T00:00:00Z\r\nContent-Type: application/http; msgtype=response\r\nContent-Length: 57\r\n\r\nHTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nsecret body\n";
371        let (table, metadata, warnings) = parse_warc(source).unwrap();
372        assert!(metadata.contains("WARC/1.1"));
373        assert_eq!(table.rows[0][2], "200");
374        assert!(table.rows[0][1].contains("token=***"));
375        assert!(warnings.iter().any(|warning| warning.contains("skipped")));
376    }
377
378    #[test]
379    fn rejects_truncated_warc_payload() {
380        let source = b"WARC/1.0\nWARC-Type: metadata\nContent-Length: 4\n\nxx";
381        assert!(parse_warc(source).is_err());
382    }
383}
document_svg/document/warc.rs

document_svg/document/
warc.rs
