Skip to main content

graphrt_doc/
text.rs

1//! Text-based formats: plain text, delimited tables, HTML, subtitles.
2
3use crate::OutputFormat;
4
5/// Decode bytes as text: BOM-aware UTF-8/UTF-16, falling back to Windows-1252
6/// (a superset of Latin-1) when the bytes are not valid UTF-8.
7pub fn decode(bytes: &[u8]) -> String {
8    if let Some((encoding, bom_len)) = encoding_rs::Encoding::for_bom(bytes) {
9        let body = bytes.get(bom_len..).unwrap_or_default();
10        let (text, _) = encoding.decode_without_bom_handling(body);
11        return text.into_owned();
12    }
13    match std::str::from_utf8(bytes) {
14        Ok(s) => s.to_owned(),
15        Err(_) => {
16            let (text, _) = encoding_rs::WINDOWS_1252.decode_without_bom_handling(bytes);
17            text.into_owned()
18        }
19    }
20}
21
22/// Parse delimited text (CSV / TSV) into rows. Handles quoted fields with embedded
23/// delimiters, doubled quotes and newlines.
24pub fn parse_delimited(text: &str, delimiter: char) -> Vec<Vec<String>> {
25    let mut rows = Vec::new();
26    let mut row = Vec::new();
27    let mut field = String::new();
28    let mut in_quotes = false;
29    let mut chars = text.chars().peekable();
30    while let Some(c) = chars.next() {
31        if in_quotes {
32            if c == '"' {
33                if chars.peek() == Some(&'"') {
34                    field.push('"');
35                    chars.next();
36                } else {
37                    in_quotes = false;
38                }
39            } else {
40                field.push(c);
41            }
42            continue;
43        }
44        match c {
45            '"' if field.is_empty() => in_quotes = true,
46            c if c == delimiter => row.push(std::mem::take(&mut field)),
47            '\r' => {}
48            '\n' => {
49                row.push(std::mem::take(&mut field));
50                rows.push(std::mem::take(&mut row));
51            }
52            c => field.push(c),
53        }
54    }
55    if !field.is_empty() || !row.is_empty() {
56        row.push(field);
57        rows.push(row);
58    }
59    rows
60}
61
62/// Render rows as text: Markdown table (first row = header) or tab-separated lines.
63///
64/// The table may not be wider than [`Options::max_cols`](crate::Options::max_cols)
65/// columns, and rendering stops as soon as the output exceeds
66/// [`Options::max_output_chars`](crate::Options::max_output_chars), so padding short
67/// rows to the table width can never multiply the output beyond that budget.
68pub fn render_table(rows: &[Vec<String>], opts: &crate::Options) -> Result<String, crate::DocError> {
69    let width = rows.iter().map(Vec::len).max().unwrap_or(0);
70    let mut table = TableWriter::new(width, opts)?;
71    for row in rows {
72        if !table.row(row.iter().map(String::as_str)) {
73            break;
74        }
75    }
76    Ok(table.finish())
77}
78
79/// Markdown table with the first row as header, using the default limits.
80/// Pipes and newlines in cells are escaped.
81pub fn markdown_table(rows: &[Vec<String>]) -> String {
82    let opts = crate::Options::default().with_format(OutputFormat::Markdown);
83    render_table(rows, &opts).unwrap_or_default()
84}
85
86/// Streaming table renderer with a running character budget.
87#[derive(Debug)]
88pub(crate) struct TableWriter {
89    out: String,
90    chars: usize,
91    budget: usize,
92    width: usize,
93    markdown: bool,
94    rows: usize,
95}
96
97impl TableWriter {
98    /// A writer for a table `width` columns wide. Fails when the table is wider than
99    /// `opts.max_cols`.
100    pub(crate) fn new(width: usize, opts: &crate::Options) -> Result<Self, crate::DocError> {
101        if width > opts.max_cols {
102            return Err(crate::DocError::LimitExceeded(format!("table has {width} columns (limit {})", opts.max_cols)));
103        }
104        Ok(Self {
105            out: String::new(),
106            chars: 0,
107            budget: opts.max_output_chars,
108            width,
109            markdown: opts.format == OutputFormat::Markdown,
110            rows: 0,
111        })
112    }
113
114    /// Append one row. Returns `false` once the output budget is exhausted; further
115    /// rows are ignored.
116    pub(crate) fn row<'a>(&mut self, cells: impl Iterator<Item = &'a str>) -> bool {
117        if self.width == 0 {
118            return true;
119        }
120        if self.chars > self.budget {
121            return false;
122        }
123        let mut line = String::new();
124        if self.markdown {
125            line.push('|');
126            let mut written = 0usize;
127            for cell in cells.take(self.width) {
128                line.push(' ');
129                for c in cell.chars() {
130                    match c {
131                        '|' => line.push_str("\\|"),
132                        '\r' | '\n' => line.push(' '),
133                        c => line.push(c),
134                    }
135                }
136                line.push_str(" |");
137                written += 1;
138            }
139            for _ in written..self.width {
140                line.push_str("  |");
141            }
142        } else {
143            let joined: Vec<&str> = cells.take(self.width).collect();
144            line.push_str(joined.join("\t").trim_end());
145        }
146        self.push_line(&line);
147        self.rows += 1;
148        if self.markdown && self.rows == 1 {
149            let separator = format!("|{}", " --- |".repeat(self.width));
150            self.push_line(&separator);
151        }
152        self.chars <= self.budget
153    }
154
155    fn push_line(&mut self, line: &str) {
156        if !self.out.is_empty() {
157            self.out.push('\n');
158            self.chars += 1;
159        }
160        self.out.push_str(line);
161        self.chars += line.chars().count();
162    }
163
164    /// The rendered table.
165    pub(crate) fn finish(self) -> String {
166        self.out
167    }
168}
169
170/// Extract the cue text of WebVTT / SRT subtitles (timings, indexes and headers removed).
171pub fn subtitles(text: &str) -> String {
172    let mut out: Vec<String> = Vec::new();
173    let mut in_note = false;
174    for raw in text.lines() {
175        let line = raw.trim().trim_start_matches('\u{feff}');
176        if line.is_empty() {
177            in_note = false;
178            continue;
179        }
180        if in_note {
181            continue;
182        }
183        if line.starts_with("WEBVTT") || line.starts_with("STYLE") || line.starts_with("REGION") {
184            in_note = true;
185            continue;
186        }
187        if line.starts_with("NOTE") {
188            in_note = true;
189            continue;
190        }
191        if line.contains("-->") || line.chars().all(|c| c.is_ascii_digit()) {
192            continue;
193        }
194        let cleaned = strip_inline_tags(line);
195        if out.last() != Some(&cleaned) && !cleaned.is_empty() {
196            out.push(cleaned);
197        }
198    }
199    out.join("\n")
200}
201
202fn strip_inline_tags(line: &str) -> String {
203    let mut out = String::with_capacity(line.len());
204    let mut in_tag = false;
205    for c in line.chars() {
206        match c {
207            '<' => in_tag = true,
208            '>' if in_tag => in_tag = false,
209            c if !in_tag => out.push(c),
210            _ => {}
211        }
212    }
213    decode_entities(out.trim())
214}
215
216/// Convert HTML to readable text. Scripts, styles and comments are dropped; block
217/// elements become line breaks. In Markdown mode headings become `#` lines and
218/// list items `- ` lines.
219pub fn html_to_text(html: &str, format: OutputFormat) -> String {
220    let markdown = format == OutputFormat::Markdown;
221    let mut out = String::with_capacity(html.len() / 2);
222    let lower = html.to_ascii_lowercase();
223    let bytes = html.as_bytes();
224    let mut i = 0;
225    let mut text_start = 0;
226    while i < bytes.len() {
227        if bytes.get(i) != Some(&b'<') {
228            i += 1;
229            continue;
230        }
231        push_text(&mut out, html.get(text_start..i).unwrap_or_default());
232        let rest = lower.get(i..).unwrap_or_default();
233        if rest.starts_with("<!--") {
234            i = rest.find("-->").map_or(bytes.len(), |p| i + p + 3);
235            text_start = i;
236            continue;
237        }
238        let Some(end) = rest.find('>') else {
239            text_start = bytes.len();
240            break;
241        };
242        let tag = rest.get(1..end).unwrap_or_default();
243        let closing = tag.starts_with('/');
244        let name: String =
245            tag.trim_start_matches('/').chars().take_while(|c| c.is_ascii_alphanumeric()).collect::<String>();
246        i += end + 1;
247        if !closing && matches!(name.as_str(), "script" | "style" | "noscript" | "template" | "head") {
248            let close = format!("</{name}");
249            i = lower.get(i..).and_then(|r| r.find(&close)).map_or(bytes.len(), |p| {
250                let after = i + p;
251                lower.get(after..).and_then(|r| r.find('>')).map_or(bytes.len(), |q| after + q + 1)
252            });
253            text_start = i;
254            continue;
255        }
256        match name.as_str() {
257            "br" => out.push('\n'),
258            "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
259                ensure_blank_line(&mut out);
260                if markdown && !closing {
261                    let level = name.get(1..).and_then(|d| d.parse::<usize>().ok()).unwrap_or(1);
262                    out.push_str(&"#".repeat(level));
263                    out.push(' ');
264                }
265            }
266            "li" if !closing => {
267                ensure_newline(&mut out);
268                if markdown {
269                    out.push_str("- ");
270                }
271            }
272            "td" | "th" if !closing && !out.ends_with('\n') && !out.is_empty() => out.push('\t'),
273            "p" | "div" | "section" | "article" | "header" | "footer" | "blockquote" | "pre" | "table" | "ul"
274            | "ol" | "nav" | "aside" | "main" | "figure" | "hr" => ensure_blank_line(&mut out),
275            "tr" | "li" | "dt" | "dd" => ensure_newline(&mut out),
276            _ => {}
277        }
278        text_start = i;
279    }
280    push_text(&mut out, html.get(text_start..).unwrap_or_default());
281    collapse_blank_lines(&out)
282}
283
284fn push_text(out: &mut String, raw: &str) {
285    if raw.is_empty() {
286        return;
287    }
288    let decoded = decode_entities(raw);
289    let mut last_space = out.ends_with([' ', '\n', '\t']) || out.is_empty();
290    for c in decoded.chars() {
291        if c.is_whitespace() {
292            if !last_space {
293                out.push(' ');
294                last_space = true;
295            }
296        } else {
297            out.push(c);
298            last_space = false;
299        }
300    }
301}
302
303fn ensure_newline(out: &mut String) {
304    trim_trailing_spaces(out);
305    if !out.is_empty() && !out.ends_with('\n') {
306        out.push('\n');
307    }
308}
309
310fn ensure_blank_line(out: &mut String) {
311    trim_trailing_spaces(out);
312    if out.is_empty() {
313        return;
314    }
315    if !out.ends_with('\n') {
316        out.push('\n');
317    }
318    if !out.ends_with("\n\n") {
319        out.push('\n');
320    }
321}
322
323fn trim_trailing_spaces(out: &mut String) {
324    while out.ends_with(' ') || out.ends_with('\t') {
325        out.pop();
326    }
327}
328
329/// Trim each line and collapse runs of blank lines to one.
330pub fn collapse_blank_lines(text: &str) -> String {
331    let mut out = Vec::new();
332    let mut blank = false;
333    for line in text.lines() {
334        let line = line.trim();
335        if line.is_empty() {
336            if !blank && !out.is_empty() {
337                out.push("");
338            }
339            blank = true;
340        } else {
341            out.push(line);
342            blank = false;
343        }
344    }
345    while out.last() == Some(&"") {
346        out.pop();
347    }
348    out.join("\n")
349}
350
351/// Decode the common named entities plus numeric character references.
352pub fn decode_entities(s: &str) -> String {
353    if !s.contains('&') {
354        return s.to_owned();
355    }
356    let mut out = String::with_capacity(s.len());
357    let mut rest = s;
358    while let Some(pos) = rest.find('&') {
359        out.push_str(rest.get(..pos).unwrap_or_default());
360        let after = rest.get(pos + 1..).unwrap_or_default();
361        // Entity names are short: only look at a bounded window, so input with many
362        // `&` and no `;` stays linear.
363        let entity_end = after.bytes().take(11).position(|b| b == b';');
364        let decoded = entity_end.and_then(|e| decode_entity(after.get(..e).unwrap_or_default()));
365        match (decoded, entity_end) {
366            (Some(c), Some(e)) => {
367                out.push(c);
368                rest = after.get(e + 1..).unwrap_or_default();
369            }
370            _ => {
371                out.push('&');
372                rest = after;
373            }
374        }
375    }
376    out.push_str(rest);
377    out
378}
379
380fn decode_entity(name: &str) -> Option<char> {
381    if let Some(num) = name.strip_prefix('#') {
382        let code = match num.strip_prefix(['x', 'X']) {
383            Some(hex) => u32::from_str_radix(hex, 16).ok()?,
384            None => num.parse().ok()?,
385        };
386        return char::from_u32(code);
387    }
388    Some(match name {
389        "amp" => '&',
390        "lt" => '<',
391        "gt" => '>',
392        "quot" => '"',
393        "apos" => '\'',
394        "nbsp" => ' ',
395        "ndash" => '–',
396        "mdash" => '—',
397        "hellip" => '…',
398        "copy" => '©',
399        "reg" => '®',
400        "trade" => '™',
401        "lsquo" => '‘',
402        "rsquo" => '’',
403        "ldquo" => '“',
404        "rdquo" => '”',
405        "bull" => '•',
406        "middot" => '·',
407        "euro" => '€',
408        _ => return None,
409    })
410}
411
412#[cfg(test)]
413mod tests {
414    use super::*;
415
416    #[test]
417    fn decodes_latin1_fallback_and_bom() {
418        assert_eq!(decode(b"caf\xe9"), "café");
419        assert_eq!(decode("\u{feff}hi".as_bytes()), "hi");
420        assert_eq!(decode(&[0xFF, 0xFE, b'h', 0, b'i', 0]), "hi");
421    }
422
423    #[test]
424    fn csv_quotes() {
425        let rows = parse_delimited("a,b\n\"x,1\",\"say \"\"hi\"\"\nthere\"\r\n", ',');
426        assert_eq!(rows, vec![vec!["a", "b"], vec!["x,1", "say \"hi\"\nthere"]]);
427        let md = markdown_table(&rows);
428        assert!(md.starts_with("| a | b |\n| --- | --- |\n"), "{md}");
429        assert!(md.contains("say \"hi\" there"));
430    }
431
432    #[test]
433    fn html_strips_scripts_and_styles() {
434        let html = "<html><head><title>T</title><style>p{}</style></head><body>\
435            <h1>Title</h1><script>alert('x')</script><p>Hello&nbsp;<b>world</b> &amp; you</p>\
436            <!-- secret --><ul><li>one</li><li>two</li></ul></body></html>";
437        let text = html_to_text(html, OutputFormat::Text);
438        assert_eq!(text, "Title\n\nHello world & you\n\none\ntwo");
439        let md = html_to_text(html, OutputFormat::Markdown);
440        assert!(md.contains("# Title"));
441        assert!(md.contains("- one\n- two"));
442        assert!(!md.contains("alert") && !md.contains("secret") && !md.contains("p{}"));
443    }
444
445    #[test]
446    fn subtitle_cues() {
447        let vtt = "WEBVTT\n\nNOTE hidden\nstill hidden\n\n1\n00:00:01.000 --> 00:00:02.000\n<v Bob>Hello</v>\n\n00:00:03.000 --> 00:00:04.000\nWorld &amp; all\n";
448        assert_eq!(subtitles(vtt), "Hello\nWorld & all");
449        let srt = "1\n00:00:01,000 --> 00:00:02,000\nHi\n\n2\n00:00:02,000 --> 00:00:03,000\nThere\n";
450        assert_eq!(subtitles(srt), "Hi\nThere");
451    }
452}