mdstream 0.2.0

Streaming-first Markdown middleware for LLM output (committed + pending blocks, render-agnostic).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

use mdstream::{
    AnalyzedStream, BlockHintAnalyzer, BlockHintMeta, BlockKind, CodeFenceAnalyzer, CodeFenceClass,
    DocumentState, FootnotesMode, IncompleteImageDropTransformer,
    IncompleteLinkPlaceholderTransformer, Options,
};

fn print_block(prefix: &str, id: u64, kind: BlockKind, text: &str) {
    let mut first = text.lines().next().unwrap_or("");
    if first.len() > 80 {
        first = &first[..80];
    }
    println!("{prefix} id={id} kind={kind:?} first_line={first:?}");
}

fn main() {
    // For demo purposes, disable terminator link/image handling and enable the built-in
    // Streamdown-compatible pending transformers instead.
    let opts = Options {
        footnotes: FootnotesMode::SingleBlock,
        terminator: mdstream::pending::TerminatorOptions {
            links: false,
            images: false,
            ..Default::default()
        },
        ..Default::default()
    };

    // Chain analyzers: (code fence meta, pending hint meta)
    let analyzer = (CodeFenceAnalyzer, BlockHintAnalyzer);
    let mut s = AnalyzedStream::new(opts, analyzer);
    let mut state = DocumentState::new();
    s.inner_mut()
        .push_pending_transformer(IncompleteLinkPlaceholderTransformer::default());
    s.inner_mut()
        .push_pending_transformer(IncompleteImageDropTransformer::default());

    let chunks = [
        "# Streaming demo\n\n",
        "Normal text with **bold",
        " continued**.\n\n",
        "See [docs](",
        " and an image ![alt](",
        "...\n\n",
        "```mermaid\n",
        "graph TD;\nA-->B;\n",
        "```\n\n",
        "After code fence.\n",
        "\n$$\nE = mc^2\n",
        "$$\n",
    ];

    for (i, chunk) in chunks.iter().enumerate() {
        println!("\n== append step {i} ==");
        let u = s.append(chunk);
        let update = u.update;

        for (block, meta) in update
            .committed
            .iter()
            .zip(u.committed_meta.iter().map(|m| &m.meta))
        {
            print_block("committed", block.id.0, block.kind, &block.raw);

            if let Some(code_meta) = &meta.0 {
                match code_meta.class {
                    CodeFenceClass::Mermaid => println!("  meta: code fence class=mermaid"),
                    CodeFenceClass::Json => println!("  meta: code fence class=json"),
                    CodeFenceClass::Other => {}
                }
            }
        }

        if let Some(p) = &update.pending {
            print_block("pending  ", p.id.0, p.kind, &p.raw);

            if let Some(pm) = &u.pending_meta {
                let hint = pm.meta.1.unwrap_or(mdstream::BlockHintMeta { flags: 0 });
                if hint.likely_incomplete() {
                    let mut flags = Vec::new();
                    if hint.has(BlockHintMeta::DISPLAY_TRANSFORMED) {
                        flags.push("display_transformed");
                    }
                    if hint.has(BlockHintMeta::UNCLOSED_CODE_FENCE) {
                        flags.push("unclosed_code_fence");
                    }
                    if hint.has(BlockHintMeta::UNBALANCED_MATH) {
                        flags.push("unbalanced_math");
                    }
                    println!("  hint: likely_incomplete flags={flags:?}");
                }
            }

            if let Some(display) = &p.display {
                if display != &p.raw {
                    println!("  display (pending only):");
                    println!("{display}");
                }
            }
        } else {
            println!("pending: <none>");
        }

        let applied = state.apply(update);
        if applied.reset {
            println!("reset: true (drop cached UI state and rebuild)");
        }

        // State view (what a UI would keep).
        println!(
            "state: committed={} pending={}",
            state.committed().len(),
            state.pending().is_some()
        );
    }

    println!("\n== finalize ==");
    let u = s.finalize();
    let update = u.update;
    for b in &update.committed {
        print_block("committed", b.id.0, b.kind, &b.raw);
    }
    state.apply(update);
    println!("pending: {:?}", state.pending().map(|b| (b.id.0, b.kind)));
}