mdstream 0.3.0

Streaming-first Markdown middleware for LLM output (committed + pending blocks, render-agnostic).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

#[derive(Debug, Default)]
pub(crate) struct FootnoteSemantics {
    detected: bool,
    scan_tail: String,
}

impl FootnoteSemantics {
    pub(crate) fn detected(&self) -> bool {
        self.detected
    }

    pub(crate) fn observe_chunk(&mut self, chunk: &str) {
        if self.detected {
            return;
        }
        if detect_footnotes(chunk) {
            self.detected = true;
            return;
        }

        // Keep a small tail window to detect patterns across chunk boundaries.
        const MAX_TAIL: usize = 256;
        let chunk_prefix = take_prefix_at_char_boundary(chunk, MAX_TAIL);
        if !self.scan_tail.is_empty() && !chunk_prefix.is_empty() {
            let mut combined = String::with_capacity(self.scan_tail.len() + chunk_prefix.len());
            combined.push_str(&self.scan_tail);
            combined.push_str(chunk_prefix);
            if detect_footnotes(&combined) {
                self.detected = true;
            }
        }
        if !self.detected {
            update_tail(&mut self.scan_tail, chunk, MAX_TAIL);
        }
    }

    pub(crate) fn reset(&mut self) {
        self.detected = false;
        self.scan_tail.clear();
    }
}

fn detect_footnotes(text: &str) -> bool {
    // Very small, streaming-friendly detector:
    // - references: [^id] (not followed by :)
    // - definitions: [^id]:
    //
    // Compatibility notes:
    // - Align with Streamdown/Incremark: identifiers must not contain whitespace, and must be non-empty.
    // - Keep a conservative identifier length cap to avoid pathological scans.
    let bytes = text.as_bytes();
    let mut i = 0usize;
    while i + 2 < bytes.len() {
        if bytes[i] == b'[' && bytes[i + 1] == b'^' {
            const MAX_ID_LEN: usize = 200;
            // Find closing `]` while validating identifier rules.
            let mut j = i + 2;
            let mut id_len = 0usize;
            while j < bytes.len() {
                let b = bytes[j];
                if b == b']' {
                    break;
                }
                if b == b'\n' || b == b'\r' || b == b' ' || b == b'\t' {
                    // Invalid footnote identifier; do not treat as footnote.
                    id_len = 0;
                    break;
                }
                id_len += 1;
                if id_len > MAX_ID_LEN {
                    id_len = 0;
                    break;
                }
                j += 1;
            }
            if id_len > 0 && j < bytes.len() && bytes[j] == b']' {
                // Either a reference (`[^id]`) or a definition (`[^id]:`) should trigger single-block mode.
                return true;
            }
        }
        i += 1;
    }
    false
}

fn take_prefix_at_char_boundary(s: &str, max_bytes: usize) -> &str {
    if s.len() <= max_bytes {
        return s;
    }
    let mut end = max_bytes;
    while end > 0 && !s.is_char_boundary(end) {
        end -= 1;
    }
    &s[..end]
}

fn take_suffix_at_char_boundary(s: &str, max_bytes: usize) -> &str {
    if s.len() <= max_bytes {
        return s;
    }
    let mut start = s.len() - max_bytes;
    while start < s.len() && !s.is_char_boundary(start) {
        start += 1;
    }
    &s[start..]
}

fn update_tail(tail: &mut String, chunk: &str, max_bytes: usize) {
    if chunk.is_empty() {
        return;
    }
    if chunk.len() >= max_bytes {
        *tail = take_suffix_at_char_boundary(chunk, max_bytes).to_string();
        return;
    }
    if tail.len() + chunk.len() <= max_bytes {
        tail.push_str(chunk);
        return;
    }
    let mut combined = String::with_capacity(max_bytes + 4);
    combined.push_str(tail);
    combined.push_str(chunk);
    *tail = take_suffix_at_char_boundary(&combined, max_bytes).to_string();
}