1use crate::OutputFormat;
4
5pub fn decode(bytes: &[u8]) -> String {
8 if let Some((encoding, bom_len)) = encoding_rs::Encoding::for_bom(bytes) {
9 let body = bytes.get(bom_len..).unwrap_or_default();
10 let (text, _) = encoding.decode_without_bom_handling(body);
11 return text.into_owned();
12 }
13 match std::str::from_utf8(bytes) {
14 Ok(s) => s.to_owned(),
15 Err(_) => {
16 let (text, _) = encoding_rs::WINDOWS_1252.decode_without_bom_handling(bytes);
17 text.into_owned()
18 }
19 }
20}
21
22pub fn parse_delimited(text: &str, delimiter: char) -> Vec<Vec<String>> {
25 let mut rows = Vec::new();
26 let mut row = Vec::new();
27 let mut field = String::new();
28 let mut in_quotes = false;
29 let mut chars = text.chars().peekable();
30 while let Some(c) = chars.next() {
31 if in_quotes {
32 if c == '"' {
33 if chars.peek() == Some(&'"') {
34 field.push('"');
35 chars.next();
36 } else {
37 in_quotes = false;
38 }
39 } else {
40 field.push(c);
41 }
42 continue;
43 }
44 match c {
45 '"' if field.is_empty() => in_quotes = true,
46 c if c == delimiter => row.push(std::mem::take(&mut field)),
47 '\r' => {}
48 '\n' => {
49 row.push(std::mem::take(&mut field));
50 rows.push(std::mem::take(&mut row));
51 }
52 c => field.push(c),
53 }
54 }
55 if !field.is_empty() || !row.is_empty() {
56 row.push(field);
57 rows.push(row);
58 }
59 rows
60}
61
62pub fn render_table(rows: &[Vec<String>], opts: &crate::Options) -> Result<String, crate::DocError> {
69 let width = rows.iter().map(Vec::len).max().unwrap_or(0);
70 let mut table = TableWriter::new(width, opts)?;
71 for row in rows {
72 if !table.row(row.iter().map(String::as_str)) {
73 break;
74 }
75 }
76 Ok(table.finish())
77}
78
79pub fn markdown_table(rows: &[Vec<String>]) -> String {
82 let opts = crate::Options::default().with_format(OutputFormat::Markdown);
83 render_table(rows, &opts).unwrap_or_default()
84}
85
86#[derive(Debug)]
88pub(crate) struct TableWriter {
89 out: String,
90 chars: usize,
91 budget: usize,
92 width: usize,
93 markdown: bool,
94 rows: usize,
95}
96
97impl TableWriter {
98 pub(crate) fn new(width: usize, opts: &crate::Options) -> Result<Self, crate::DocError> {
101 if width > opts.max_cols {
102 return Err(crate::DocError::LimitExceeded(format!("table has {width} columns (limit {})", opts.max_cols)));
103 }
104 Ok(Self {
105 out: String::new(),
106 chars: 0,
107 budget: opts.max_output_chars,
108 width,
109 markdown: opts.format == OutputFormat::Markdown,
110 rows: 0,
111 })
112 }
113
114 pub(crate) fn row<'a>(&mut self, cells: impl Iterator<Item = &'a str>) -> bool {
117 if self.width == 0 {
118 return true;
119 }
120 if self.chars > self.budget {
121 return false;
122 }
123 let mut line = String::new();
124 if self.markdown {
125 line.push('|');
126 let mut written = 0usize;
127 for cell in cells.take(self.width) {
128 line.push(' ');
129 for c in cell.chars() {
130 match c {
131 '|' => line.push_str("\\|"),
132 '\r' | '\n' => line.push(' '),
133 c => line.push(c),
134 }
135 }
136 line.push_str(" |");
137 written += 1;
138 }
139 for _ in written..self.width {
140 line.push_str(" |");
141 }
142 } else {
143 let joined: Vec<&str> = cells.take(self.width).collect();
144 line.push_str(joined.join("\t").trim_end());
145 }
146 self.push_line(&line);
147 self.rows += 1;
148 if self.markdown && self.rows == 1 {
149 let separator = format!("|{}", " --- |".repeat(self.width));
150 self.push_line(&separator);
151 }
152 self.chars <= self.budget
153 }
154
155 fn push_line(&mut self, line: &str) {
156 if !self.out.is_empty() {
157 self.out.push('\n');
158 self.chars += 1;
159 }
160 self.out.push_str(line);
161 self.chars += line.chars().count();
162 }
163
164 pub(crate) fn finish(self) -> String {
166 self.out
167 }
168}
169
170pub fn subtitles(text: &str) -> String {
172 let mut out: Vec<String> = Vec::new();
173 let mut in_note = false;
174 for raw in text.lines() {
175 let line = raw.trim().trim_start_matches('\u{feff}');
176 if line.is_empty() {
177 in_note = false;
178 continue;
179 }
180 if in_note {
181 continue;
182 }
183 if line.starts_with("WEBVTT") || line.starts_with("STYLE") || line.starts_with("REGION") {
184 in_note = true;
185 continue;
186 }
187 if line.starts_with("NOTE") {
188 in_note = true;
189 continue;
190 }
191 if line.contains("-->") || line.chars().all(|c| c.is_ascii_digit()) {
192 continue;
193 }
194 let cleaned = strip_inline_tags(line);
195 if out.last() != Some(&cleaned) && !cleaned.is_empty() {
196 out.push(cleaned);
197 }
198 }
199 out.join("\n")
200}
201
202fn strip_inline_tags(line: &str) -> String {
203 let mut out = String::with_capacity(line.len());
204 let mut in_tag = false;
205 for c in line.chars() {
206 match c {
207 '<' => in_tag = true,
208 '>' if in_tag => in_tag = false,
209 c if !in_tag => out.push(c),
210 _ => {}
211 }
212 }
213 decode_entities(out.trim())
214}
215
216pub fn html_to_text(html: &str, format: OutputFormat) -> String {
220 let markdown = format == OutputFormat::Markdown;
221 let mut out = String::with_capacity(html.len() / 2);
222 let lower = html.to_ascii_lowercase();
223 let bytes = html.as_bytes();
224 let mut i = 0;
225 let mut text_start = 0;
226 while i < bytes.len() {
227 if bytes.get(i) != Some(&b'<') {
228 i += 1;
229 continue;
230 }
231 push_text(&mut out, html.get(text_start..i).unwrap_or_default());
232 let rest = lower.get(i..).unwrap_or_default();
233 if rest.starts_with("<!--") {
234 i = rest.find("-->").map_or(bytes.len(), |p| i + p + 3);
235 text_start = i;
236 continue;
237 }
238 let Some(end) = rest.find('>') else {
239 text_start = bytes.len();
240 break;
241 };
242 let tag = rest.get(1..end).unwrap_or_default();
243 let closing = tag.starts_with('/');
244 let name: String =
245 tag.trim_start_matches('/').chars().take_while(|c| c.is_ascii_alphanumeric()).collect::<String>();
246 i += end + 1;
247 if !closing && matches!(name.as_str(), "script" | "style" | "noscript" | "template" | "head") {
248 let close = format!("</{name}");
249 i = lower.get(i..).and_then(|r| r.find(&close)).map_or(bytes.len(), |p| {
250 let after = i + p;
251 lower.get(after..).and_then(|r| r.find('>')).map_or(bytes.len(), |q| after + q + 1)
252 });
253 text_start = i;
254 continue;
255 }
256 match name.as_str() {
257 "br" => out.push('\n'),
258 "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
259 ensure_blank_line(&mut out);
260 if markdown && !closing {
261 let level = name.get(1..).and_then(|d| d.parse::<usize>().ok()).unwrap_or(1);
262 out.push_str(&"#".repeat(level));
263 out.push(' ');
264 }
265 }
266 "li" if !closing => {
267 ensure_newline(&mut out);
268 if markdown {
269 out.push_str("- ");
270 }
271 }
272 "td" | "th" if !closing && !out.ends_with('\n') && !out.is_empty() => out.push('\t'),
273 "p" | "div" | "section" | "article" | "header" | "footer" | "blockquote" | "pre" | "table" | "ul"
274 | "ol" | "nav" | "aside" | "main" | "figure" | "hr" => ensure_blank_line(&mut out),
275 "tr" | "li" | "dt" | "dd" => ensure_newline(&mut out),
276 _ => {}
277 }
278 text_start = i;
279 }
280 push_text(&mut out, html.get(text_start..).unwrap_or_default());
281 collapse_blank_lines(&out)
282}
283
284fn push_text(out: &mut String, raw: &str) {
285 if raw.is_empty() {
286 return;
287 }
288 let decoded = decode_entities(raw);
289 let mut last_space = out.ends_with([' ', '\n', '\t']) || out.is_empty();
290 for c in decoded.chars() {
291 if c.is_whitespace() {
292 if !last_space {
293 out.push(' ');
294 last_space = true;
295 }
296 } else {
297 out.push(c);
298 last_space = false;
299 }
300 }
301}
302
303fn ensure_newline(out: &mut String) {
304 trim_trailing_spaces(out);
305 if !out.is_empty() && !out.ends_with('\n') {
306 out.push('\n');
307 }
308}
309
310fn ensure_blank_line(out: &mut String) {
311 trim_trailing_spaces(out);
312 if out.is_empty() {
313 return;
314 }
315 if !out.ends_with('\n') {
316 out.push('\n');
317 }
318 if !out.ends_with("\n\n") {
319 out.push('\n');
320 }
321}
322
323fn trim_trailing_spaces(out: &mut String) {
324 while out.ends_with(' ') || out.ends_with('\t') {
325 out.pop();
326 }
327}
328
329pub fn collapse_blank_lines(text: &str) -> String {
331 let mut out = Vec::new();
332 let mut blank = false;
333 for line in text.lines() {
334 let line = line.trim();
335 if line.is_empty() {
336 if !blank && !out.is_empty() {
337 out.push("");
338 }
339 blank = true;
340 } else {
341 out.push(line);
342 blank = false;
343 }
344 }
345 while out.last() == Some(&"") {
346 out.pop();
347 }
348 out.join("\n")
349}
350
351pub fn decode_entities(s: &str) -> String {
353 if !s.contains('&') {
354 return s.to_owned();
355 }
356 let mut out = String::with_capacity(s.len());
357 let mut rest = s;
358 while let Some(pos) = rest.find('&') {
359 out.push_str(rest.get(..pos).unwrap_or_default());
360 let after = rest.get(pos + 1..).unwrap_or_default();
361 let entity_end = after.bytes().take(11).position(|b| b == b';');
364 let decoded = entity_end.and_then(|e| decode_entity(after.get(..e).unwrap_or_default()));
365 match (decoded, entity_end) {
366 (Some(c), Some(e)) => {
367 out.push(c);
368 rest = after.get(e + 1..).unwrap_or_default();
369 }
370 _ => {
371 out.push('&');
372 rest = after;
373 }
374 }
375 }
376 out.push_str(rest);
377 out
378}
379
380fn decode_entity(name: &str) -> Option<char> {
381 if let Some(num) = name.strip_prefix('#') {
382 let code = match num.strip_prefix(['x', 'X']) {
383 Some(hex) => u32::from_str_radix(hex, 16).ok()?,
384 None => num.parse().ok()?,
385 };
386 return char::from_u32(code);
387 }
388 Some(match name {
389 "amp" => '&',
390 "lt" => '<',
391 "gt" => '>',
392 "quot" => '"',
393 "apos" => '\'',
394 "nbsp" => ' ',
395 "ndash" => '–',
396 "mdash" => '—',
397 "hellip" => '…',
398 "copy" => '©',
399 "reg" => '®',
400 "trade" => '™',
401 "lsquo" => '‘',
402 "rsquo" => '’',
403 "ldquo" => '“',
404 "rdquo" => '”',
405 "bull" => '•',
406 "middot" => '·',
407 "euro" => '€',
408 _ => return None,
409 })
410}
411
412#[cfg(test)]
413mod tests {
414 use super::*;
415
416 #[test]
417 fn decodes_latin1_fallback_and_bom() {
418 assert_eq!(decode(b"caf\xe9"), "café");
419 assert_eq!(decode("\u{feff}hi".as_bytes()), "hi");
420 assert_eq!(decode(&[0xFF, 0xFE, b'h', 0, b'i', 0]), "hi");
421 }
422
423 #[test]
424 fn csv_quotes() {
425 let rows = parse_delimited("a,b\n\"x,1\",\"say \"\"hi\"\"\nthere\"\r\n", ',');
426 assert_eq!(rows, vec![vec!["a", "b"], vec!["x,1", "say \"hi\"\nthere"]]);
427 let md = markdown_table(&rows);
428 assert!(md.starts_with("| a | b |\n| --- | --- |\n"), "{md}");
429 assert!(md.contains("say \"hi\" there"));
430 }
431
432 #[test]
433 fn html_strips_scripts_and_styles() {
434 let html = "<html><head><title>T</title><style>p{}</style></head><body>\
435 <h1>Title</h1><script>alert('x')</script><p>Hello <b>world</b> & you</p>\
436 <!-- secret --><ul><li>one</li><li>two</li></ul></body></html>";
437 let text = html_to_text(html, OutputFormat::Text);
438 assert_eq!(text, "Title\n\nHello world & you\n\none\ntwo");
439 let md = html_to_text(html, OutputFormat::Markdown);
440 assert!(md.contains("# Title"));
441 assert!(md.contains("- one\n- two"));
442 assert!(!md.contains("alert") && !md.contains("secret") && !md.contains("p{}"));
443 }
444
445 #[test]
446 fn subtitle_cues() {
447 let vtt = "WEBVTT\n\nNOTE hidden\nstill hidden\n\n1\n00:00:01.000 --> 00:00:02.000\n<v Bob>Hello</v>\n\n00:00:03.000 --> 00:00:04.000\nWorld & all\n";
448 assert_eq!(subtitles(vtt), "Hello\nWorld & all");
449 let srt = "1\n00:00:01,000 --> 00:00:02,000\nHi\n\n2\n00:00:02,000 --> 00:00:03,000\nThere\n";
450 assert_eq!(subtitles(srt), "Hi\nThere");
451 }
452}