mdstream 0.3.0

Streaming-first Markdown middleware for LLM output (committed + pending blocks, render-agnostic).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

mod support;

use mdstream::{
    AnalyzedStream, ContainerBoundaryPlugin, FenceBoundaryPlugin, MdStream, Options,
    TagBoundaryPlugin, TaggedBlockAnalyzer,
};

#[test]
fn tag_boundary_accepts_up_to_three_leading_spaces() {
    let markdown = "Intro\n\n   <thinking>\nA\n   </thinking>\n\nAfter\n";
    let blocks = support::collect_final_raw_with_stream(
        support::chunk_whole(markdown),
        MdStream::new(Options::default()).with_boundary_plugin(TagBoundaryPlugin::thinking()),
    );

    assert_eq!(
        blocks,
        vec![
            "Intro\n\n".to_string(),
            "   <thinking>\nA\n   </thinking>\n".to_string(),
            "After\n".to_string(),
        ]
    );
}

#[test]
fn tag_boundary_does_not_accept_four_leading_spaces() {
    let markdown = "Intro\n\n    <thinking>\nA\n    </thinking>\n\nAfter\n";
    let blocks = support::collect_final_raw_with_stream(
        support::chunk_whole(markdown),
        MdStream::new(Options::default()).with_boundary_plugin(TagBoundaryPlugin::thinking()),
    );

    assert_eq!(
        blocks,
        vec![
            "Intro\n\n".to_string(),
            "    <thinking>\nA\n    </thinking>\n\n".to_string(),
            "After\n".to_string(),
        ]
    );
}

#[test]
fn tagged_block_analyzer_does_not_analyze_indented_code_like_tag_block() {
    let mut s = AnalyzedStream::new(Options::default(), TaggedBlockAnalyzer::default());
    let u = s.append("    <thinking>\nA\n    </thinking>\n");

    assert!(u.committed_meta.is_empty());
    assert!(u.pending_meta.is_none());

    let u = s.finalize();
    assert!(u.committed_meta.is_empty());
    assert!(u.pending_meta.is_none());
}

#[test]
fn tagged_block_analyzer_treats_trailing_closing_text_as_open() {
    let mut s = AnalyzedStream::new(Options::default(), TaggedBlockAnalyzer::default());
    let u = s.append("<thinking>\nA\n</thinking> trailing\n");
    let meta = u.committed_meta.into_iter().next().expect("tag meta").meta;

    assert_eq!(meta.tag, "thinking");
    assert!(!meta.closed);
    assert!(meta.content.contains("</thinking> trailing"));
}

#[test]
fn directive_container_and_fence_container_keep_distinct_start_syntax() {
    let directive = "Intro\n\n::: warning\nA\n:::\n\nAfter\n";
    let fence = "Intro\n\n:::warning\nA\n:::\n\nAfter\n";

    let directive_blocks = support::collect_final_raw_with_stream(
        support::chunk_whole(directive),
        MdStream::new(Options::default()).with_boundary_plugin(ContainerBoundaryPlugin::default()),
    );
    let fence_blocks = support::collect_final_raw_with_stream(
        support::chunk_whole(fence),
        MdStream::new(Options::default()).with_boundary_plugin(FenceBoundaryPlugin::triple_colon()),
    );
    let directive_plugin_on_fence_syntax = support::collect_final_raw_with_stream(
        support::chunk_whole(fence),
        MdStream::new(Options::default()).with_boundary_plugin(ContainerBoundaryPlugin::default()),
    );

    assert_eq!(directive_blocks[1], "::: warning\nA\n:::\n");
    assert_eq!(fence_blocks[1], ":::warning\nA\n:::\n");
    assert_eq!(
        directive_plugin_on_fence_syntax,
        vec![
            "Intro\n\n".to_string(),
            ":::warning\nA\n:::\n\n".to_string(),
            "After\n".to_string()
        ]
    );
}

#[test]
fn directive_container_nested_longer_markers_stay_in_one_block() {
    let markdown = ":::: outer\nA\n:::::: inner\nB\n::::\nC\n::::\nAfter\n";
    let blocks = support::collect_final_raw_with_stream(
        support::chunk_whole(markdown),
        MdStream::new(Options::default()).with_boundary_plugin(ContainerBoundaryPlugin::default()),
    );

    assert_eq!(
        blocks,
        vec![
            ":::: outer\nA\n:::::: inner\nB\n::::\nC\n::::\n".to_string(),
            "After\n".to_string(),
        ]
    );
}