mdstream 0.2.0

Streaming-first Markdown middleware for LLM output (committed + pending blocks, render-agnostic).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

mod support;

use mdstream::Options;

#[test]
fn streamdown_benchmark_single_block() {
    let markdown =
        include_str!("fixtures/streamdown_bench/basic_single_block.md").trim_end_matches('\n');

    let opts = Options::default();
    let blocks_whole = support::collect_final_raw(support::chunk_whole(markdown), opts.clone());
    let blocks_lines = support::collect_final_raw(support::chunk_lines(markdown), opts.clone());
    let blocks_rand = support::collect_final_raw(
        support::chunk_pseudo_random(markdown, "streamdown_benchmark_single_block", 0, 40),
        opts.clone(),
    );

    assert_eq!(blocks_lines, blocks_whole);
    assert_eq!(blocks_rand, blocks_whole);

    assert_eq!(
        blocks_whole,
        vec![
            "# Heading\n".to_string(),
            "This is a paragraph.".to_string()
        ]
    );
}

#[test]
fn streamdown_benchmark_multiple_blocks_10() {
    // From Streamdown's parse-blocks benchmark ("multiple blocks (10)").
    let markdown = include_str!("fixtures/streamdown_bench/basic_multiple_blocks_10.md");

    let opts = Options::default();
    let blocks_whole = support::collect_final_raw(support::chunk_whole(markdown), opts.clone());
    let blocks_lines = support::collect_final_raw(support::chunk_lines(markdown), opts.clone());
    let blocks_rand = support::collect_final_raw(
        support::chunk_pseudo_random(markdown, "streamdown_benchmark_multiple_blocks_10", 0, 40),
        opts.clone(),
    );

    assert_eq!(blocks_lines, blocks_whole);
    assert_eq!(blocks_rand, blocks_whole);

    // Sanity checks: key blocks exist and order is stable.
    assert!(blocks_whole.iter().any(|b| b == "# Heading 1\n"));
    assert!(
        blocks_whole
            .iter()
            .any(|b| b.contains("This is paragraph 1."))
    );
    assert!(blocks_whole.iter().any(|b| b == "## Heading 2\n"));
    assert!(blocks_whole.iter().any(|b| b.contains("- List item 1")));
    assert!(blocks_whole.iter().any(|b| b.contains("> Blockquote text")));
}

#[test]
fn streamdown_benchmark_many_blocks_100() {
    // From Streamdown's parse-blocks benchmark ("many blocks (100)").
    let markdown = include_str!("fixtures/streamdown_bench/basic_many_blocks_100.md");

    let opts = Options::default();
    let blocks_whole = support::collect_final_raw(support::chunk_whole(markdown), opts.clone());
    let blocks_lines = support::collect_final_raw(support::chunk_lines(markdown), opts.clone());
    let blocks_rand = support::collect_final_raw(
        support::chunk_pseudo_random(markdown, "streamdown_benchmark_many_blocks_100", 0, 40),
        opts.clone(),
    );

    assert_eq!(blocks_lines, blocks_whole);
    assert_eq!(blocks_rand, blocks_whole);

    // Each section yields one heading block and one paragraph block.
    assert_eq!(blocks_whole.len(), 200);
    assert_eq!(blocks_whole[0], "## Section 0\n");
    assert!(blocks_whole[1].starts_with("Paragraph 0"));
    assert!(blocks_whole[blocks_whole.len() - 2].starts_with("## Section 99"));
    assert!(blocks_whole[blocks_whole.len() - 1].starts_with("Paragraph 99"));
}