lex-core 0.8.5

Parser library for the lex format
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134

//! Property-based tests for the lex parser
//!
//! These tests use the proptest framework to generate random inputs and verify
//! that the parser behaves correctly (e.g. never panics) across a wide range
//! of inputs, including valid, invalid, and pathological cases.

use lex_core::lex::parsing::parse_document;
use proptest::prelude::*;

// Property-based tests using the strategy
proptest! {
    // We limit cases to avoid extremely long test runs in CI, but
    // it can be increased for deep local fuzzing
    #![proptest_config(ProptestConfig::with_cases(250))]

    #[test]
    fn test_parse_document_never_panics_on_any_utf8(s in "\\PC*") {
        // The parser should never panic on any valid UTF-8 string.
        // It might return an error or parse it as plain text, but it must not crash.
        let _ = parse_document(&s);
    }
}
fn lex_text_strategy() -> impl Strategy<Value = String> {
    prop::collection::vec(
        prop_oneof![
            "[a-zA-Z0-9]+",
            "[a-zA-Z0-9]+ [a-zA-Z0-9]+",
            "[a-zA-Z0-9]+[.,!?]",
            "",
        ],
        1..10,
    )
    .prop_map(|lines| lines.join("\n"))
}

fn list_item_strategy() -> impl Strategy<Value = String> {
    prop_oneof![
        "- [a-zA-Z0-9 ]+",
        "[0-9]+\\. [a-zA-Z0-9 ]+",
        "[a-z]\\. [a-zA-Z0-9 ]+",
        "\\([0-9]+\\) [a-zA-Z0-9 ]+",
    ]
}

fn session_title_strategy() -> impl Strategy<Value = String> {
    prop_oneof!["[0-9]+\\. [a-zA-Z0-9 ]+", "[a-zA-Z0-9 ]+:", "[a-zA-Z0-9 ]+",]
}

fn lex_document_strategy() -> impl Strategy<Value = String> {
    prop::collection::vec(
        prop_oneof![
            lex_text_strategy(),
            list_item_strategy(),
            session_title_strategy(),
        ],
        1..20,
    )
    .prop_map(|lines| lines.join("\n"))
}

proptest! {
    #![proptest_config(ProptestConfig::with_cases(250))]

    #[test]
    fn test_parse_document_never_panics_on_valid_looking_lex(input in lex_document_strategy()) {
        let _ = parse_document(&input);
    }
}

/// Generate valid Lex verbatim blocks: Subject:\n    content\n:: label ::
fn verbatim_block_strategy() -> impl Strategy<Value = String> {
    (
        "[A-Z][a-zA-Z0-9 ]{1,15}",
        prop::collection::vec("[a-zA-Z0-9 ]+", 1..5),
        "[a-z][a-z0-9_-]{0,8}",
    )
        .prop_map(|(subject, lines, label)| {
            let subject = subject.trim_end().to_string();
            let content = lines
                .iter()
                .map(|l| format!("    {l}"))
                .collect::<Vec<_>>()
                .join("\n");
            format!("{subject}:\n{content}\n:: {label} ::")
        })
}

/// Generate valid Lex annotations: :: label :: or :: label key=value ::
fn annotation_strategy() -> impl Strategy<Value = String> {
    prop_oneof![
        // Marker annotation
        "[a-z][a-z0-9_-]{0,8}".prop_map(|label| format!(":: {label} ::")),
        // Annotation with parameter
        ("[a-z][a-z0-9_-]{0,8}", "[a-z][a-z0-9_]{0,6}", "[a-z0-9]+",)
            .prop_map(|(label, key, value)| format!(":: {label} {key}={value} ::")),
    ]
}

/// Generate valid Lex inline text (using actual Lex inline syntax)
fn inline_text_strategy() -> impl Strategy<Value = String> {
    prop_oneof![
        "This is *[a-z]+* text.",
        "This is _[a-z]+_ text.",
        "This is `[a-z]+` text.",
        "This has #[a-z]+# math.",
        "See \\[@[a-z]+\\] here.",
        "Mixed *[a-z]+* and _[a-z]+_ text.",
        "Reference \\[\\^[a-z]+\\] note.",
    ]
}

fn expanded_lex_document_strategy() -> impl Strategy<Value = String> {
    prop::collection::vec(
        prop_oneof![
            lex_text_strategy(),
            list_item_strategy(),
            session_title_strategy(),
            verbatim_block_strategy(),
            annotation_strategy(),
            inline_text_strategy(),
        ],
        1..30,
    )
    .prop_map(|lines| lines.join("\n\n")) // Join with double newlines to ensure separation
}

proptest! {
    #![proptest_config(ProptestConfig::with_cases(250))]

    #[test]
    fn test_parse_document_never_panics_on_expanded_lex(input in expanded_lex_document_strategy()) {
        let _ = parse_document(&input);
    }
}