nab 0.8.2

Token-optimized HTTP client for LLMs — fetches any URL as clean markdown
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

//! Diff strategies for comparing watch snapshot bodies.
//!
//! All strategies operate on UTF-8 text (HTML is decoded before hashing).
//! The primary concern for watch polling is content-addressed hashing —
//! the actual diff text shown to users is kept simple (line count changes).

use sha2::{Digest, Sha256};

use super::types::DiffKind;

// ─── Hash ─────────────────────────────────────────────────────────────────────

/// Compute the SHA-256 hex digest of `body`.
pub fn sha256_hex(body: &[u8]) -> String {
    let mut hasher = Sha256::new();
    hasher.update(body);
    hex::encode(hasher.finalize())
}

// ─── Content extraction ───────────────────────────────────────────────────────

/// Extract the text to be hashed from `html`, applying `selector` if given.
///
/// For `DiffKind::Dom` / `DiffKind::Semantic` with a selector, only the
/// matched subtree is extracted. Falls back to full-page text on any error.
pub fn extract_content(html: &str, selector: Option<&str>, kind: &DiffKind) -> String {
    match (selector, kind) {
        (Some(sel), DiffKind::Dom | DiffKind::Text | DiffKind::Semantic) => {
            extract_selector(html, sel).unwrap_or_else(|| html.to_owned())
        }
        _ => html.to_owned(),
    }
}

/// Extract the inner text of the first element matching `selector`.
fn extract_selector(html: &str, selector: &str) -> Option<String> {
    use scraper::{Html, Selector};
    let doc = Html::parse_document(html);
    let sel = Selector::parse(selector).ok()?;
    doc.select(&sel).next().map(|el| el.html())
}

// ─── Diff summary ─────────────────────────────────────────────────────────────

/// Produce a human-readable one-line summary of the change.
///
/// Currently reports the number of lines added/removed relative to the old body.
/// Future work: full semantic or DOM diffing.
pub fn diff_summary(old: &str, new: &str) -> String {
    let old_lines = old.lines().count();
    let new_lines = new.lines().count();
    match new_lines.cmp(&old_lines) {
        std::cmp::Ordering::Greater => {
            format!("content grew by {} lines", new_lines - old_lines)
        }
        std::cmp::Ordering::Less => {
            format!("content shrank by {} lines", old_lines - new_lines)
        }
        std::cmp::Ordering::Equal => "content changed (same line count)".into(),
    }
}

// ─── Tests ────────────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn text_diff_detects_change() {
        // GIVEN: two different bodies
        let a = b"hello world";
        let b = b"hello rust";
        // WHEN: hashed
        let ha = sha256_hex(a);
        let hb = sha256_hex(b);
        // THEN: hashes differ
        assert_ne!(ha, hb);
    }

    #[test]
    fn identical_bodies_produce_same_hash() {
        // GIVEN: identical bytes
        let body = b"same content";
        assert_eq!(sha256_hex(body), sha256_hex(body));
    }

    #[test]
    fn selector_diff_isolates_subtree() {
        // GIVEN: HTML with a specific section
        let html = r#"<html><body><div id="target"><p>price: $99</p></div><footer>noise</footer></body></html>"#;
        // WHEN: extract with selector
        let extracted = extract_content(html, Some("#target"), &DiffKind::Dom);
        // THEN: contains target, does not contain noise
        assert!(extracted.contains("price"), "got: {extracted}");
        assert!(!extracted.contains("noise"), "got: {extracted}");
    }

    #[test]
    fn selector_fallback_on_miss() {
        // GIVEN: HTML without the selector
        let html = "<html><body><p>no match</p></body></html>";
        // WHEN: selector doesn't match
        let extracted = extract_content(html, Some("#nonexistent"), &DiffKind::Dom);
        // THEN: falls back to full page
        assert!(extracted.contains("no match"), "got: {extracted}");
    }

    #[test]
    fn diff_summary_growing() {
        let old = "a\nb\nc";
        let new = "a\nb\nc\nd\ne";
        let s = diff_summary(old, new);
        assert!(s.contains("grew"), "got: {s}");
    }

    #[test]
    fn diff_summary_shrinking() {
        let old = "a\nb\nc\nd";
        let new = "a\nb";
        let s = diff_summary(old, new);
        assert!(s.contains("shrank"), "got: {s}");
    }
}