mdstream 0.2.0

Streaming-first Markdown middleware for LLM output (committed + pending blocks, render-agnostic).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CodeFenceHeader<'a> {
    pub fence_char: char,
    pub fence_len: usize,
    /// Entire info string (trimmed), excluding fence markers.
    pub info: &'a str,
    /// First token of `info`, lowercased if ASCII. Empty means "no language".
    pub language: Option<&'a str>,
}

fn is_space_or_tab(b: u8) -> bool {
    b == b' ' || b == b'\t'
}

pub fn parse_code_fence_header(line: &str) -> Option<CodeFenceHeader<'_>> {
    // CommonMark-ish fence opening line:
    // - up to 3 leading spaces
    // - fence is ``` or ~~~ (>=3)
    // - info string is the rest of the line after the fence run
    let mut s = line;
    let mut spaces = 0usize;
    while spaces < 3 && s.starts_with(' ') {
        s = &s[1..];
        spaces += 1;
    }

    let bytes = s.as_bytes();
    if bytes.len() < 3 {
        return None;
    }
    let fence_char = bytes[0] as char;
    if fence_char != '`' && fence_char != '~' {
        return None;
    }
    let mut fence_len = 0usize;
    while fence_len < bytes.len() && bytes[fence_len] == bytes[0] {
        fence_len += 1;
    }
    if fence_len < 3 {
        return None;
    }

    let info = s[fence_len..].trim();
    let language = info
        .split_whitespace()
        .next()
        .and_then(|tok| if tok.is_empty() { None } else { Some(tok) });

    Some(CodeFenceHeader {
        fence_char,
        fence_len,
        info,
        language,
    })
}

pub fn parse_code_fence_header_from_block(text: &str) -> Option<CodeFenceHeader<'_>> {
    let first_line = text.split('\n').next().unwrap_or(text);
    parse_code_fence_header(first_line)
}

pub fn is_code_fence_closing_line(line: &str, fence_char: char, fence_len: usize) -> bool {
    // Mirrors `src/stream.rs` fence_end behavior, but exported for consumers.
    let mut s = line;
    let mut spaces = 0usize;
    while spaces < 3 && s.starts_with(' ') {
        s = &s[1..];
        spaces += 1;
    }
    let trimmed = s.trim_end();
    let mut count = 0usize;
    for ch in trimmed.chars() {
        if ch != fence_char {
            return false;
        }
        count += 1;
    }
    count >= fence_len
}

pub fn is_list_marker_line_prefix(line: &str) -> bool {
    // Equivalent to remend listItemPattern: /^[\s]*[-*+][\s]+$/
    // This is exposed for adapters that want to replicate remend-like heuristics.
    let bytes = line.as_bytes();
    let mut i = 0usize;
    while i < bytes.len() && is_space_or_tab(bytes[i]) {
        i += 1;
    }
    if i >= bytes.len() {
        return false;
    }
    let marker = bytes[i];
    if marker != b'-' && marker != b'*' && marker != b'+' {
        return false;
    }
    i += 1;
    if i >= bytes.len() {
        return false;
    }
    let mut has_ws = false;
    while i < bytes.len() {
        if is_space_or_tab(bytes[i]) {
            has_ws = true;
            i += 1;
            continue;
        }
        return false;
    }
    has_ws
}