booky 0.8.0

A tool to analyze English text
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

// Basic HTML content parsing
use anyhow::Result;
use core::str;
use html_escape::decode_html_entities;
use std::io::{BufRead, stdin};

struct Content {
    /// read buffer
    buf: Vec<u8>,
    /// DOM element stack display bool
    stack: Vec<bool>,
}

fn main() -> Result<()> {
    let mut content = Content::new();
    content.parse(stdin().lock())?;
    Ok(())
}

const NON_CLOSING: &[&str] = &["!--", "img", "input", "link", "meta", "source"];

const NON_DISPLAYED: &[&str] = &[
    "a",
    "annotation",
    "figure",
    "footer",
    "form",
    "header",
    "label",
    "nav",
    "script",
    "semantics",
    "style",
];

impl Content {
    fn new() -> Self {
        Content {
            buf: Vec::with_capacity(4096),
            stack: Vec::new(),
        }
    }

    fn is_content_displayed(&self) -> bool {
        !self.stack.iter().any(|d| !d)
    }

    fn handle_content(&self) {
        if self.is_content_displayed() {
            if let Ok(text) = str::from_utf8(&self.buf) {
                let text = text.trim();
                if !text.is_empty() {
                    println!("{}", decode_html_entities(text));
                }
            }
        }
    }

    fn parse<R>(&mut self, mut reader: R) -> Result<()>
    where
        R: BufRead,
    {
        loop {
            self.buf.clear();
            if reader.read_until(b'<', &mut self.buf)? == 0 {
                break;
            }
            if let Some(b'<') = self.buf.pop() {
                self.handle_content();
                self.buf.clear();
                if reader.read_until(b'>', &mut self.buf)? == 0 {
                    break;
                }
                if let Some(b'>') = self.buf.pop() {
                    if let Ok(text) = str::from_utf8(&self.buf) {
                        let mut parts = text.trim().split_whitespace();
                        if let Some(elem) = parts.next() {
                            if NON_CLOSING.contains(&elem) {
                                continue;
                            }
                            if elem.starts_with('/') {
                                self.stack.pop();
                            } else {
                                let displayed = !NON_DISPLAYED.contains(&elem);
                                let hidden = parts.any(is_class_hidden);
                                self.stack.push(displayed && !hidden);
                            }
                        }
                    }
                }
            }
        }
        Ok(())
    }
}

fn is_class_hidden(part: &str) -> bool {
    part.starts_with("class") && {
        part.contains("catlinks")
            || part.contains("sidebar-list")
            || part.contains("infobox")
            || part.contains("vector-menu")
            || part.contains("references")
    }
}