mdstream 0.3.0

Streaming-first Markdown middleware for LLM output (committed + pending blocks, render-agnostic).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

#![cfg(feature = "pulldown")]

use mdstream::adapters::pulldown::{PulldownAdapter, PulldownAdapterOptions};
use mdstream::{
    Block, BlockId, BlockKind, BlockStatus, MdStream, Options, ReferenceDefinitionsMode,
};
use pulldown_cmark::{Event, Tag};

fn contains_link(events: &[Event<'static>]) -> bool {
    events
        .iter()
        .any(|e| matches!(e, Event::Start(Tag::Link { .. })))
}

#[test]
fn pulldown_adapter_reparses_invalidated_blocks_with_reference_definitions() {
    let opts = Options {
        reference_definitions: ReferenceDefinitionsMode::Invalidate,
        ..Default::default()
    };

    let mut s = MdStream::new(opts);
    let mut a = PulldownAdapter::new(PulldownAdapterOptions::default());

    // Start paragraph with a shortcut reference.
    s.append("See [ref].\n\n");

    // Starting the definition line commits the previous paragraph block.
    let u1 = s.append("[ref]: https://example.com\n");
    a.apply_update(&u1);
    let block1_id = u1.committed[0].id;

    let e1 = a.committed_events(block1_id).expect("events");
    assert!(!contains_link(e1), "definition not committed yet");

    // Commit the definition block, which should invalidate block 1.
    s.append("\n");
    let u2 = s.append("Next\n");
    a.apply_update(&u2);

    assert!(u2.invalidated.contains(&block1_id));
    let e2 = a.committed_events(block1_id).expect("events");
    assert!(
        contains_link(e2),
        "block should be re-parsed with definitions"
    );
}

#[test]
fn pulldown_adapter_uses_latest_definition_for_repeated_labels() {
    let mut a = PulldownAdapter::new(PulldownAdapterOptions::default());
    let update = mdstream::Update {
        committed: vec![Block {
            id: BlockId(1),
            status: BlockStatus::Committed,
            kind: BlockKind::Paragraph,
            raw: "[ref]: https://old.example\n[ref]: https://new.example\n".to_string(),
            display: None,
        }],
        pending: None,
        reset: false,
        invalidated: Vec::new(),
    };
    a.apply_update(&update);

    let pending = Block {
        id: BlockId(2),
        status: BlockStatus::Pending,
        kind: BlockKind::Paragraph,
        raw: "See [ref].".to_string(),
        display: None,
    };
    let events = a.parse_pending(&pending);

    assert!(
        events.iter().any(|event| match event {
            Event::Start(Tag::Link { dest_url, .. }) => dest_url.as_ref() == "https://new.example",
            _ => false,
        }),
        "pending parse should use the latest definition text"
    );
}

#[test]
fn pulldown_adapter_pending_parse_uses_committed_definition_prelude() {
    let mut a = PulldownAdapter::new(PulldownAdapterOptions::default());
    let update = mdstream::Update {
        committed: vec![Block {
            id: BlockId(1),
            status: BlockStatus::Committed,
            kind: BlockKind::Paragraph,
            raw: "[ref]: https://example.com\n".to_string(),
            display: None,
        }],
        pending: None,
        reset: false,
        invalidated: Vec::new(),
    };
    a.apply_update(&update);

    let pending = Block {
        id: BlockId(2),
        status: BlockStatus::Pending,
        kind: BlockKind::Paragraph,
        raw: "See [ref].".to_string(),
        display: None,
    };

    assert!(contains_link(&a.parse_pending(&pending)));
}