kibble 0.1.0

chew through any source into clean datasets — a fast ingestion, RAG & fine-tuning toolkit
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176

use scraper::{Html, Selector};

/// Extract readable main text from an HTML document.
/// Prefers <article>/<main>; ignores script/style/nav/header/footer; emits block text.
pub fn extract_main_text(html: &str) -> String {
    let doc = Html::parse_document(html);

    let root = ["article", "main", "body"].iter().find_map(|tag| {
        let sel = Selector::parse(tag).ok()?;
        doc.select(&sel).next()
    });
    let Some(root) = root else {
        return String::new();
    };

    let skip = ["script", "style", "noscript", "nav", "header", "footer"];
    let block_tags = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "pre"];
    let block_sel = Selector::parse("p, h1, h2, h3, h4, h5, h6, li, blockquote, pre").unwrap();

    let mut blocks: Vec<String> = Vec::new();
    for el in root.select(&block_sel) {
        // Walk ancestors: exclude if under a noise tag, or nested inside another block tag.
        let mut excluded = false;
        let mut cur = el.parent();
        while let Some(node) = cur {
            if let Some(e) = scraper::ElementRef::wrap(node) {
                let name = e.value().name();
                if skip.contains(&name) || block_tags.contains(&name) {
                    excluded = true;
                    break;
                }
            }
            cur = node.parent();
        }
        if excluded {
            continue;
        }
        let block = if el.value().name() == "pre" {
            // Preserve internal whitespace/indentation; trim only surrounding blank lines.
            el.text().collect::<String>().trim_matches('\n').trim_end().to_string()
        } else {
            el.text().collect::<Vec<_>>().join(" ").split_whitespace().collect::<Vec<_>>().join(" ")
        };
        if !block.is_empty() {
            blocks.push(block);
        }
    }
    blocks.join("\n\n")
}

/// GET a URL and return the response body text, erroring on transport or HTTP status failure.
pub async fn fetch_url(client: &reqwest::Client, url: &str) -> std::io::Result<String> {
    let resp = client
        .get(url)
        .send()
        .await
        .and_then(|r| r.error_for_status())
        .map_err(|e| std::io::Error::other(format!("fetch failed for {url}: {e}")))?;
    resp.text()
        .await
        .map_err(|e| std::io::Error::other(format!("read body failed for {url}: {e}")))
}

/// Fetch a URL, reading at most `max_bytes` of the body (streamed cap), lossy-decoded to text.
pub async fn fetch_url_capped(client: &reqwest::Client, url: &str, max_bytes: u64) -> std::io::Result<String> {
    let mut resp = client.get(url).send().await.map_err(|e| std::io::Error::other(format!("fetch {url}: {e}")))?;
    let mut buf: Vec<u8> = Vec::new();
    while let Some(chunk) = resp.chunk().await.map_err(|e| std::io::Error::other(format!("read {url}: {e}")))? {
        let room = (max_bytes as usize).saturating_sub(buf.len());
        if room == 0 { break; }
        buf.extend_from_slice(&chunk[..room.min(chunk.len())]);
        if buf.len() as u64 >= max_bytes { break; }
    }
    Ok(String::from_utf8_lossy(&buf).into_owned())
}

#[cfg(test)]
mod tests {
    use super::extract_main_text;
    use super::fetch_url_capped;

    #[tokio::test]
    async fn fetch_url_gets_body_from_localhost() {
        use std::io::{Read, Write};
        use std::net::TcpListener;

        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let port = listener.local_addr().unwrap().port();
        let handle = std::thread::spawn(move || {
            if let Ok((mut stream, _)) = listener.accept() {
                let mut buf = [0u8; 1024];
                let _ = stream.read(&mut buf); // consume request
                let body = "<html><body><p>Fetched body text.</p></body></html>";
                let resp = format!(
                    "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: text/html\r\n\r\n{}",
                    body.len(),
                    body
                );
                let _ = stream.write_all(resp.as_bytes());
                let _ = stream.flush();
            }
        });

        let client = crate::net::build_client(None).unwrap();
        let url = format!("http://127.0.0.1:{port}/");
        let html = super::fetch_url(&client, &url).await.unwrap();
        handle.join().unwrap();
        assert!(html.contains("Fetched body text."));
    }

    #[tokio::test]
    async fn fetch_url_errors_on_bad_host() {
        let client = crate::net::build_client(None).unwrap();
        // 127.0.0.1:1 — nothing listening; connection refused.
        let res = super::fetch_url(&client, "http://127.0.0.1:1/").await;
        assert!(res.is_err());
    }

    #[test]
    fn extracts_article_text_and_drops_noise() {
        let html = r#"
            <html><head><style>.x{}</style><title>t</title></head>
            <body>
              <nav>menu junk</nav>
              <article>
                <h1>The Title</h1>
                <p>Hello   world.</p>
                <script>evil()</script>
                <p>Second paragraph.</p>
              </article>
              <footer>footer junk</footer>
            </body></html>
        "#;
        let out = extract_main_text(html);
        assert_eq!(out, "The Title\n\nHello world.\n\nSecond paragraph.");
    }

    #[test]
    fn falls_back_to_body_without_article() {
        let html = "<html><body><p>Only body.</p><p>Two.</p></body></html>";
        assert_eq!(extract_main_text(html), "Only body.\n\nTwo.");
    }

    #[test]
    fn empty_on_no_text() {
        assert_eq!(extract_main_text("<html><body></body></html>"), "");
    }

    #[test]
    fn preserves_pre_whitespace() {
        let html = "<html><body><article><pre>fn main() {\n    let x = 1;\n}</pre></article></body></html>";
        let out = super::extract_main_text(html);
        assert_eq!(out, "fn main() {\n    let x = 1;\n}");
    }

    #[test]
    fn dedupes_nested_block_text() {
        let html = "<html><body><article><blockquote><p>quoted line</p></blockquote></article></body></html>";
        // blockquote text already contains the <p> text; must appear once, not twice
        assert_eq!(super::extract_main_text(html), "quoted line");
    }

    #[tokio::test]
    async fn fetch_url_capped_caps_bytes() {
        use std::io::{Read, Write};
        use std::net::TcpListener;
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();
        let big = "x".repeat(10_000);
        let resp = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{}", big.len(), big);
        std::thread::spawn(move || { if let Ok((mut s,_)) = listener.accept() { let mut b=[0u8;1024]; let _=s.read(&mut b); let _=s.write_all(resp.as_bytes()); }});
        let client = crate::net::build_client(None).unwrap();
        let body = fetch_url_capped(&client, &format!("http://{addr}/"), 100).await.unwrap();
        assert!(body.len() <= 100, "capped, got {}", body.len());
    }
}