rig-core 0.44.0

An opinionated library for building LLM powered applications.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

use crate::loaders::epub::{RawTextProcessor, StripXmlProcessor, TextProcessor};
use crate::loaders::test_fixtures::{fixture_glob, fixture_path};

use super::EpubFileLoader;

#[test]
fn test_epub_loader_with_ignoring_errors() {
    let glob = fixture_glob("*.epub");
    let loader = EpubFileLoader::<_, RawTextProcessor>::with_glob(&glob).unwrap();
    let actual = loader
        .load_with_path()
        .ignore_errors()
        .by_chapter()
        .ignore_errors()
        .into_iter()
        .collect::<Vec<_>>();

    assert_eq!(actual.len(), 1);

    let (_, chapters) = &actual[0];
    assert_eq!(chapters.len(), 3);
}

#[test]
fn test_single_file() {
    let glob = fixture_glob("*.epub");
    let loader = EpubFileLoader::<_, RawTextProcessor>::with_glob(&glob).unwrap();

    let actual = loader
        .read()
        .ignore_errors()
        .into_iter()
        .collect::<Vec<_>>();

    assert_eq!(actual.len(), 1);
}

#[test]
fn test_single_file_with_path() {
    let glob = fixture_glob("*.epub");
    let loader = EpubFileLoader::<_, RawTextProcessor>::with_glob(&glob).unwrap();

    let actual = loader
        .read_with_path()
        .ignore_errors()
        .into_iter()
        .collect::<Vec<_>>();

    assert_eq!(actual.len(), 1);

    let (path, _) = &actual[0];
    assert_eq!(path, &fixture_path("dummy.epub"));
}

fn assert_stripped_xml(xml: &str, expected: &str) {
    let actual = StripXmlProcessor::process(xml);
    assert!(
        matches!(&actual, Ok(text) if text == expected),
        "input {xml:?}: expected {expected:?}, got {actual:?}"
    );
}

#[test]
fn stripped_xml_preserves_entity_text() {
    assert_stripped_xml("<p>A&amp;B&#x21;</p>", "A&B!");
}

#[test]
fn stripped_xml_resolves_predefined_and_adjacent_references_once() {
    for (xml, expected) in [
        ("<p>&lt;&gt;&amp;&apos;&quot;</p>", "<>&'\""),
        ("<p>&amp;&#38;&#x26;</p>", "&&&"),
        ("<p>&amp;lt;&amp;#33;</p>", "&lt;&#33;"),
        ("&amp;&#33;", "&!"),
    ] {
        assert_stripped_xml(xml, expected);
    }
}

#[test]
fn stripped_xml_resolves_decimal_and_hex_unicode_references() {
    for (xml, expected) in [
        ("<p>&#33;&#233;&#128512;</p>", "!é😀"),
        ("<p>&#x21;&#xE9;&#x1F600;</p>", "!é😀"),
        ("<p>é&#128512;終</p>", "é😀終"),
    ] {
        assert_stripped_xml(xml, expected);
    }
}

#[test]
fn stripped_xml_preserves_whitespace_around_references() {
    for (xml, expected) in [
        ("<p> &amp; </p>", " & "),
        ("<p>&amp; \t\n &lt;</p>", "& \t\n <"),
        ("<p>A &amp; B&#33; </p>", "A & B! "),
        ("<p>&#32;A&#x20;</p>", " A "),
        ("<p>A&#9;&#xA;B</p>", "A\t\nB"),
        ("<p> \t\n </p>", ""),
        ("<p> &#32;&#x9; </p>", ""),
    ] {
        assert_stripped_xml(xml, expected);
    }
}

#[test]
fn stripped_xml_preserves_cdata_literals_and_boundaries() {
    for (xml, expected) in [
        (
            "<p><![CDATA[&amp;&#33;&unknown;]]></p>",
            "&amp;&#33;&unknown;",
        ),
        ("<p>A<![CDATA[B]]>C</p>", "A B C"),
        ("<p>A&amp;<![CDATA[B]]>&lt;C</p>", "A& B <C"),
        ("<p><![CDATA[A]]><![CDATA[B]]></p>", "A B"),
        ("<p>A<![CDATA[ ]]>&amp;B</p>", "A &B"),
        ("<p><![CDATA[A]]> &amp; </p>", "A  & "),
    ] {
        assert_stripped_xml(xml, expected);
    }
}

#[test]
fn stripped_xml_preserves_markup_boundaries() {
    for (xml, expected) in [
        ("<p>A<b>B</b>C</p><p>D</p>", "ABCD"),
        ("<p>A&amp;<b>B&#33;</b>C</p>", "A&B!C"),
        ("<p>A<br/>&amp;B</p>", "A&B"),
        ("<p>A<!-- comment -->&amp;B</p>", "A&B"),
        ("<p>A<?note value?>&amp;B</p>", "A&B"),
        ("<p>A<b> </b>&amp;B</p>", "A&B"),
    ] {
        assert_stripped_xml(xml, expected);
    }
}