mdstream 0.2.0

Streaming-first Markdown middleware for LLM output (committed + pending blocks, render-agnostic).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

use mdstream::{BlockKind, MdStream, Options};

fn snapshot_kinds_and_raw(s: &mut MdStream) -> Vec<(BlockKind, String)> {
    s.snapshot_blocks()
        .into_iter()
        .map(|b| (b.kind, b.raw))
        .collect()
}

#[test]
fn streamdown_benchmark_streaming_text_steps_match_full_parse_per_step() {
    // Matches Streamdown's parse-blocks benchmark ("Streaming Simulation").
    let base = "# Heading\n\n";
    let delta = "This is streaming text. ";

    let opts = Options::default();
    let mut incremental = MdStream::new(opts.clone());
    incremental.append(base);

    for i in 0..50 {
        if i > 0 {
            incremental.append(delta);
        }

        let step = format!("{base}{}", delta.repeat(i));
        let mut scratch = MdStream::new(opts.clone());
        scratch.append(&step);

        let incremental_snapshot = snapshot_kinds_and_raw(&mut incremental);
        let scratch_snapshot = snapshot_kinds_and_raw(&mut scratch);
        assert_eq!(incremental_snapshot, scratch_snapshot, "step {i} mismatch");
    }
}

#[test]
fn streamdown_benchmark_streaming_code_steps_match_full_parse_per_step() {
    // Matches Streamdown's parse-blocks benchmark ("Streaming Simulation").
    let steps = [
        "```javascript",
        "```javascript\n",
        "```javascript\nconst",
        "```javascript\nconst x",
        "```javascript\nconst x =",
        "```javascript\nconst x = 1",
        "```javascript\nconst x = 1;",
        "```javascript\nconst x = 1;\n",
        "```javascript\nconst x = 1;\n```",
    ];

    let opts = Options::default();
    let mut incremental = MdStream::new(opts.clone());

    let mut prev = "";
    for (i, step) in steps.iter().enumerate() {
        let delta = step.strip_prefix(prev).expect("step must extend previous");
        incremental.append(delta);

        let mut scratch = MdStream::new(opts.clone());
        scratch.append(step);

        let incremental_snapshot = snapshot_kinds_and_raw(&mut incremental);
        let scratch_snapshot = snapshot_kinds_and_raw(&mut scratch);
        assert_eq!(incremental_snapshot, scratch_snapshot, "step {i} mismatch");

        prev = step;
    }
}