apif_ast/
gctf_tokenizer.rs

1//! Universal GCTF file tokenizer.
2//!
3//! This is the **only** module that reads raw `.gctf` text.
4//! Pipeline: `text → tokenize_gctf() → Vec<GctfToken> → parser → AST`
5//!
6//! IMPLEMENTATION NOTES:
7//! - No regex, no starts_with, no ends_with, no contains, no find()
8//! - Pure byte-level scanning with exact span tracking
9//! - Only uses .as_bytes() once per line to get byte slice for scanning
10//! - Only uses .to_string() to create owned output (necessary for API)
11//! - All "parsing" done via byte comparisons (bytes[pos] == b'-')
12//!
13//! String operations count: 0 (only .to_string() for output ownership)
14
15use crate::tokenizer::Span;
16
17#[derive(Debug, Clone, PartialEq)]
18pub struct GctfToken {
19    pub kind: GctfTokenKind,
20    pub line: usize,
21    pub span: Span,
22}
23
24#[derive(Debug, Clone, PartialEq)]
25pub enum GctfTokenKind {
26    SectionHeader { name: String, raw_options: String },
27    Comment(String),
28    Blank,
29    Content(String),
30    AttributeBlock(String),
31}
32
33pub fn tokenize_gctf(source: &str) -> Vec<GctfToken> {
34    let mut tokens = Vec::new();
35    for (line_idx, line) in source.lines().enumerate() {
36        tokens.push(classify_line(line_idx, line));
37    }
38    tokens
39}
40
41fn classify_line(line_idx: usize, line: &str) -> GctfToken {
42    let bytes = line.as_bytes();
43    let len = bytes.len();
44    let mut pos = 0;
45
46    while pos < len && is_ws(bytes[pos]) {
47        pos += 1;
48    }
49
50    if pos == len {
51        return GctfToken {
52            kind: GctfTokenKind::Blank,
53            line: line_idx,
54            span: Span { start: 0, end: len },
55        };
56    }
57
58    if pos + 1 < len && bytes[pos] == b'#' && bytes[pos + 1] == b'[' {
59        return scan_attribute_block(line_idx, line, bytes, len);
60    }
61
62    if bytes[pos] == b'#' {
63        return GctfToken {
64            kind: GctfTokenKind::Comment(line.to_string()),
65            line: line_idx,
66            span: Span { start: 0, end: len },
67        };
68    }
69
70    if pos + 1 < len && bytes[pos] == b'/' && bytes[pos + 1] == b'/' {
71        return GctfToken {
72            kind: GctfTokenKind::Comment(line.to_string()),
73            line: line_idx,
74            span: Span { start: 0, end: len },
75        };
76    }
77
78    if let Some(header) = scan_section_header(line_idx, line, bytes, len) {
79        return header;
80    }
81
82    GctfToken {
83        kind: GctfTokenKind::Content(line.to_string()),
84        line: line_idx,
85        span: Span { start: 0, end: len },
86    }
87}
88
89fn scan_attribute_block(line_idx: usize, line: &str, bytes: &[u8], len: usize) -> GctfToken {
90    let mut pos = 0;
91
92    while pos < len && is_ws(bytes[pos]) {
93        pos += 1;
94    }
95
96    if pos + 1 >= len || bytes[pos] != b'#' || bytes[pos + 1] != b'[' {
97        return GctfToken {
98            kind: GctfTokenKind::Content(line.to_string()),
99            line: line_idx,
100            span: Span { start: 0, end: len },
101        };
102    }
103
104    pos += 2;
105
106    let content_start = pos;
107
108    while pos < len && bytes[pos] != b']' {
109        pos += 1;
110    }
111
112    let content_end = pos;
113
114    let content = if content_end > content_start {
115        slice_str(line, content_start, content_end)
116    } else {
117        String::new()
118    };
119
120    GctfToken {
121        kind: GctfTokenKind::AttributeBlock(content),
122        line: line_idx,
123        span: Span { start: 0, end: len },
124    }
125}
126
127fn scan_section_header(line_idx: usize, line: &str, bytes: &[u8], len: usize) -> Option<GctfToken> {
128    let mut pos = 0;
129
130    while pos < len && is_ws(bytes[pos]) {
131        pos += 1;
132    }
133
134    if pos + 2 >= len || bytes[pos] != b'-' || bytes[pos + 1] != b'-' || bytes[pos + 2] != b'-' {
135        return None;
136    }
137    pos += 3;
138
139    while pos < len && is_ws(bytes[pos]) {
140        pos += 1;
141    }
142
143    let name_start = pos;
144    while pos < len && is_section_name_char(bytes[pos]) {
145        pos += 1;
146    }
147    if pos == name_start {
148        return None;
149    }
150    let name = slice_str(line, name_start, pos);
151
152    let mut trailing = len;
153    while trailing > pos && is_ws(bytes[trailing - 1]) {
154        trailing -= 1;
155    }
156    if trailing < pos + 3
157        || bytes[trailing - 3] != b'-'
158        || bytes[trailing - 2] != b'-'
159        || bytes[trailing - 1] != b'-'
160    {
161        return None;
162    }
163    let options_end = trailing - 3;
164
165    let mut opts_start = pos;
166    while opts_start < options_end && is_ws(bytes[opts_start]) {
167        opts_start += 1;
168    }
169    let mut opts_end = options_end;
170    while opts_end > opts_start && is_ws(bytes[opts_end - 1]) {
171        opts_end -= 1;
172    }
173
174    let raw_options = if opts_start < opts_end {
175        slice_str(line, opts_start, opts_end)
176    } else {
177        String::new()
178    };
179
180    Some(GctfToken {
181        kind: GctfTokenKind::SectionHeader { name, raw_options },
182        line: line_idx,
183        span: Span { start: 0, end: len },
184    })
185}
186
187pub fn tokenize_kv_line(line: &str) -> Option<(String, String)> {
188    let bytes = line.as_bytes();
189    let len = bytes.len();
190    let mut pos = 0;
191
192    while pos < len && is_ws(bytes[pos]) {
193        pos += 1;
194    }
195
196    if pos == len {
197        return None;
198    }
199    if bytes[pos] == b'#' {
200        return None;
201    }
202    if pos + 1 < len && bytes[pos] == b'/' && bytes[pos + 1] == b'/' {
203        return None;
204    }
205
206    let key_start = pos;
207    while pos < len && bytes[pos] != b':' {
208        pos += 1;
209    }
210    if pos == len {
211        return None;
212    }
213
214    let mut key_end = pos;
215    while key_end > key_start && is_ws(bytes[key_end - 1]) {
216        key_end -= 1;
217    }
218    pos += 1;
219
220    while pos < len && is_ws(bytes[pos]) {
221        pos += 1;
222    }
223
224    let val_start = pos;
225    let mut val_end = len;
226    while val_end > val_start && is_ws(bytes[val_end - 1]) {
227        val_end -= 1;
228    }
229
230    Some((
231        slice_str(line, key_start, key_end),
232        slice_str(line, val_start, val_end),
233    ))
234}
235
236pub fn tokenize_extract_line(line: &str) -> Option<(String, String)> {
237    let (name, _, value) = tokenize_extract_line_full(line)?;
238    Some((name, value))
239}
240
241/// Like `tokenize_extract_line` but also returns the optional type annotation
242/// from `name:Type = value` syntax.
243pub fn tokenize_extract_line_full(line: &str) -> Option<(String, Option<String>, String)> {
244    let bytes = line.as_bytes();
245    let len = bytes.len();
246    let mut pos = 0;
247
248    while pos < len && is_ws(bytes[pos]) {
249        pos += 1;
250    }
251
252    if pos == len {
253        return None;
254    }
255    if bytes[pos] == b'#' {
256        return None;
257    }
258    if pos + 1 < len && bytes[pos] == b'/' && bytes[pos + 1] == b'/' {
259        return None;
260    }
261
262    let name_start = pos;
263    while pos < len && bytes[pos] != b'=' {
264        pos += 1;
265    }
266    if pos == len {
267        return None;
268    }
269
270    let mut name_end = pos;
271    while name_end > name_start && is_ws(bytes[name_end - 1]) {
272        name_end -= 1;
273    }
274    let raw_name = slice_str(line, name_start, name_end);
275    pos += 1;
276
277    while pos < len && is_ws(bytes[pos]) {
278        pos += 1;
279    }
280
281    let val_start = pos;
282    let mut val_end = len;
283    while val_end > val_start && is_ws(bytes[val_end - 1]) {
284        val_end -= 1;
285    }
286    let value = slice_str(line, val_start, val_end);
287
288    // Parse `name:Type` pattern from the variable name
289    let (name, type_annotation) = if let Some(colon_pos) = raw_name.rfind(':') {
290        let maybe_type = raw_name[colon_pos + 1..].trim();
291        if !maybe_type.is_empty()
292            && maybe_type
293                .chars()
294                .all(|c| c.is_ascii_alphanumeric() || c == '_')
295        {
296            (
297                raw_name[..colon_pos].trim().to_string(),
298                Some(maybe_type.to_string()),
299            )
300        } else {
301            (raw_name, None)
302        }
303    } else {
304        (raw_name, None)
305    };
306
307    Some((name, type_annotation, value))
308}
309
310pub fn tokenize_inline_options(raw: &str) -> Vec<(String, String)> {
311    let bytes = raw.as_bytes();
312    let len = bytes.len();
313    let mut pos = 0;
314    let mut result = Vec::new();
315
316    while pos < len {
317        while pos < len && is_ws(bytes[pos]) {
318            pos += 1;
319        }
320        if pos >= len {
321            break;
322        }
323
324        let tok_start = pos;
325        let mut in_quotes = false;
326        let mut escaped = false;
327
328        while pos < len {
329            if escaped {
330                escaped = false;
331                pos += 1;
332                continue;
333            }
334            match bytes[pos] {
335                b'\\' => {
336                    escaped = true;
337                    pos += 1;
338                }
339                b'"' => {
340                    in_quotes = !in_quotes;
341                    pos += 1;
342                }
343                b' ' | b'\t' if !in_quotes => break,
344                _ => pos += 1,
345            }
346        }
347        let tok_end = pos;
348
349        let token = slice_str(raw, tok_start, tok_end);
350
351        let mut eq_pos = None;
352        let tb = token.as_bytes();
353        for (i, &b) in tb.iter().enumerate() {
354            if b == b'=' {
355                eq_pos = Some(i);
356                break;
357            }
358        }
359
360        if let Some(eq) = eq_pos {
361            let mut key_end = eq;
362            while key_end > 0 && is_ws(tb[key_end - 1]) {
363                key_end -= 1;
364            }
365            let mut val_start = eq + 1;
366            while val_start < tb.len() && is_ws(tb[val_start]) {
367                val_start += 1;
368            }
369            let mut val_end = tb.len();
370            while val_end > val_start && is_ws(tb[val_end - 1]) {
371                val_end -= 1;
372            }
373
374            let mut key = slice_str(&token, 0, key_end);
375            let mut value = slice_str(&token, val_start, val_end);
376
377            strip_outer_quotes(&mut key);
378            strip_outer_quotes(&mut value);
379
380            result.push((key, value));
381        } else {
382            let mut key = token;
383            strip_outer_quotes(&mut key);
384            result.push((key, "true".to_string()));
385        }
386    }
387
388    result
389}
390
391fn strip_outer_quotes(s: &mut String) {
392    if s.len() >= 2 {
393        let b = s.as_bytes();
394        if (b[0] == b'"' && b[s.len() - 1] == b'"') || (b[0] == b'\'' && b[s.len() - 1] == b'\'') {
395            *s = s[1..s.len() - 1].to_string();
396        }
397    }
398}
399
400#[inline]
401fn is_ws(b: u8) -> bool {
402    matches!(b, b' ' | b'\t')
403}
404
405#[inline]
406fn is_section_name_char(b: u8) -> bool {
407    b.is_ascii_uppercase() || b == b'_'
408}
409
410#[inline]
411fn slice_str(s: &str, start: usize, end: usize) -> String {
412    if start >= end {
413        return String::new();
414    }
415    s[start..end].to_string()
416}
417
418#[cfg(test)]
419mod tests {
420    use super::*;
421
422    #[test]
423    fn test_tokenize_empty() {
424        let tokens = tokenize_gctf("");
425        assert!(tokens.is_empty());
426    }
427
428    #[test]
429    fn test_tokenize_blank_lines() {
430        let tokens = tokenize_gctf("\n\n  \n");
431        assert_eq!(tokens.len(), 3);
432        assert!(matches!(tokens[0].kind, GctfTokenKind::Blank));
433        assert!(matches!(tokens[1].kind, GctfTokenKind::Blank));
434        assert!(matches!(tokens[2].kind, GctfTokenKind::Blank));
435    }
436
437    #[test]
438    fn test_tokenize_comments() {
439        let tokens = tokenize_gctf("# hello\n// world");
440        assert_eq!(tokens.len(), 2);
441        assert!(matches!(&tokens[0].kind, GctfTokenKind::Comment(t) if t == "# hello"));
442        assert!(matches!(&tokens[1].kind, GctfTokenKind::Comment(t) if t == "// world"));
443    }
444
445    #[test]
446    fn test_tokenize_section_headers() {
447        let tokens = tokenize_gctf("--- ENDPOINT ---\n--- RESPONSE partial=true ---");
448        assert_eq!(tokens.len(), 2);
449
450        match &tokens[0].kind {
451            GctfTokenKind::SectionHeader { name, raw_options } => {
452                assert_eq!(name, "ENDPOINT");
453                assert_eq!(raw_options, "");
454            }
455            _ => panic!("expected SectionHeader"),
456        }
457
458        match &tokens[1].kind {
459            GctfTokenKind::SectionHeader { name, raw_options } => {
460                assert_eq!(name, "RESPONSE");
461                assert_eq!(raw_options, "partial=true");
462            }
463            _ => panic!("expected SectionHeader"),
464        }
465    }
466
467    #[test]
468    fn test_tokenize_content() {
469        let tokens = tokenize_gctf("hello world\n{\"key\": \"value\"}");
470        assert_eq!(tokens.len(), 2);
471        assert!(matches!(&tokens[0].kind, GctfTokenKind::Content(t) if t == "hello world"));
472        assert!(
473            matches!(&tokens[1].kind, GctfTokenKind::Content(t) if t == "{\"key\": \"value\"}")
474        );
475    }
476
477    #[test]
478    fn test_tokenize_not_section_header() {
479        let tokens = tokenize_gctf("--- not uppercase ---\n---ABC");
480        assert!(matches!(tokens[0].kind, GctfTokenKind::Content(_)));
481        assert!(matches!(tokens[1].kind, GctfTokenKind::Content(_)));
482    }
483
484    #[test]
485    fn test_tokenize_full_document() {
486        let input = r#"--- ENDPOINT ---
487test.Service/Method
488
489--- REQUEST ---
490{}
491
492--- ASSERTS ---
493.x == 1
494"#;
495        let tokens = tokenize_gctf(input);
496        let mut kinds: Vec<&str> = Vec::new();
497        for t in &tokens {
498            match &t.kind {
499                GctfTokenKind::SectionHeader { .. } => kinds.push("H"),
500                GctfTokenKind::Comment(_) => kinds.push("C"),
501                GctfTokenKind::Blank => kinds.push("_"),
502                GctfTokenKind::Content(_) => kinds.push("T"),
503                GctfTokenKind::AttributeBlock(_) => kinds.push("A"),
504            }
505        }
506        assert_eq!(kinds, vec!["H", "T", "_", "H", "T", "_", "H", "T"]);
507    }
508
509    #[test]
510    fn test_tokenize_section_header_with_multiple_options() {
511        let tokens = tokenize_gctf("--- RESPONSE partial=true tolerance=0.1 ---");
512        match &tokens[0].kind {
513            GctfTokenKind::SectionHeader { name, raw_options } => {
514                assert_eq!(name, "RESPONSE");
515                assert_eq!(raw_options, "partial=true tolerance=0.1");
516            }
517            _ => panic!("expected SectionHeader"),
518        }
519    }
520
521    #[test]
522    fn test_kv_line_basic() {
523        let (key, value) = tokenize_kv_line("Authorization: Bearer token").unwrap();
524        assert_eq!(key, "Authorization");
525        assert_eq!(value, "Bearer token");
526    }
527
528    #[test]
529    fn test_kv_line_with_whitespace() {
530        let (key, value) = tokenize_kv_line("  key  :  value  ").unwrap();
531        assert_eq!(key, "key");
532        assert_eq!(value, "value");
533    }
534
535    #[test]
536    fn test_kv_line_comment() {
537        assert_eq!(tokenize_kv_line("# comment"), None);
538        assert_eq!(tokenize_kv_line("// comment"), None);
539    }
540
541    #[test]
542    fn test_kv_line_empty() {
543        assert_eq!(tokenize_kv_line(""), None);
544        assert_eq!(tokenize_kv_line("   "), None);
545    }
546
547    #[test]
548    fn test_kv_line_no_colon() {
549        assert_eq!(tokenize_kv_line("no colon here"), None);
550    }
551
552    #[test]
553    fn test_extract_line_basic() {
554        let (name, value) = tokenize_extract_line("total = .response.total").unwrap();
555        assert_eq!(name, "total");
556        assert_eq!(value, ".response.total");
557    }
558
559    #[test]
560    fn test_extract_line_comment() {
561        assert_eq!(tokenize_extract_line("# comment"), None);
562        assert_eq!(tokenize_extract_line("// comment"), None);
563    }
564
565    #[test]
566    fn test_extract_line_empty() {
567        assert_eq!(tokenize_extract_line(""), None);
568    }
569
570    #[test]
571    fn test_tokenize_inline_options_basic() {
572        let opts = tokenize_inline_options("key1=value1 key2=value2");
573        assert_eq!(opts.len(), 2);
574        assert_eq!(opts[0], ("key1".into(), "value1".into()));
575        assert_eq!(opts[1], ("key2".into(), "value2".into()));
576    }
577
578    #[test]
579    fn test_tokenize_inline_options_quoted() {
580        let opts = tokenize_inline_options(r#"key="hello world""#);
581        assert_eq!(opts.len(), 1);
582        assert_eq!(opts[0], ("key".into(), "hello world".into()));
583    }
584
585    #[test]
586    fn test_tokenize_inline_options_boolean_short_form() {
587        let opts = tokenize_inline_options("partial");
588        assert_eq!(opts.len(), 1);
589        assert_eq!(opts[0], ("partial".into(), "true".into()));
590    }
591
592    #[test]
593    fn test_tokenize_inline_options_complex() {
594        let opts = tokenize_inline_options("with_asserts=true partial=false tolerance=0.1");
595        assert_eq!(opts.len(), 3);
596        assert_eq!(opts[0], ("with_asserts".into(), "true".into()));
597        assert_eq!(opts[1], ("partial".into(), "false".into()));
598        assert_eq!(opts[2], ("tolerance".into(), "0.1".into()));
599    }
600
601    #[test]
602    fn test_line_numbers() {
603        let tokens = tokenize_gctf("line0\nline1\nline2");
604        assert_eq!(tokens[0].line, 0);
605        assert_eq!(tokens[1].line, 1);
606        assert_eq!(tokens[2].line, 2);
607    }
608
609    // === scan_section_header edge cases ===
610
611    #[test]
612    fn test_section_header_meta() {
613        let tokens = tokenize_gctf("--- META ---\nname: Test");
614        let GctfTokenKind::SectionHeader { name, .. } = &tokens[0].kind else {
615            panic!()
616        };
617        assert_eq!(name, "META");
618    }
619
620    #[test]
621    fn test_section_header_no_closing_dashes() {
622        let tokens = tokenize_gctf("--- ENDPOINT");
623        assert!(matches!(tokens[0].kind, GctfTokenKind::Content(_)));
624    }
625
626    #[test]
627    fn test_section_header_only_dashes() {
628        let tokens = tokenize_gctf("------");
629        assert!(matches!(tokens[0].kind, GctfTokenKind::Content(_)));
630    }
631
632    #[test]
633    fn test_section_header_leading_whitespace() {
634        let tokens = tokenize_gctf("  --- ENDPOINT ---");
635        match &tokens[0].kind {
636            GctfTokenKind::SectionHeader { name, .. } => assert_eq!(name, "ENDPOINT"),
637            _ => panic!("expected SectionHeader"),
638        }
639    }
640
641    #[test]
642    fn test_section_header_extra_whitespace() {
643        let tokens = tokenize_gctf("---   RESPONSE   partial=true   ---");
644        match &tokens[0].kind {
645            GctfTokenKind::SectionHeader { name, raw_options } => {
646                assert_eq!(name, "RESPONSE");
647                assert_eq!(raw_options, "partial=true");
648            }
649            _ => panic!("expected SectionHeader"),
650        }
651    }
652
653    #[test]
654    fn test_section_header_lowercase_rejected() {
655        let tokens = tokenize_gctf("--- endpoint ---");
656        assert!(matches!(tokens[0].kind, GctfTokenKind::Content(_)));
657    }
658
659    #[test]
660    fn test_section_header_mixed_case_treated_as_partial_name() {
661        let tokens = tokenize_gctf("--- Endpoint ---");
662        match &tokens[0].kind {
663            GctfTokenKind::SectionHeader { name, .. } => assert_eq!(name, "E"),
664            _ => panic!("expected SectionHeader with truncated name"),
665        }
666    }
667
668    #[test]
669    fn test_section_header_fully_lowercase_rejected() {
670        let tokens = tokenize_gctf("--- endpoint ---");
671        assert!(matches!(tokens[0].kind, GctfTokenKind::Content(_)));
672    }
673
674    #[test]
675    fn test_section_header_with_underscore() {
676        let tokens = tokenize_gctf("--- REQUEST_HEADERS ---");
677        match &tokens[0].kind {
678            GctfTokenKind::SectionHeader { name, .. } => assert_eq!(name, "REQUEST_HEADERS"),
679            _ => panic!("expected SectionHeader"),
680        }
681    }
682
683    #[test]
684    fn test_three_dashes_in_content() {
685        let tokens = tokenize_gctf("---ABC");
686        assert!(matches!(tokens[0].kind, GctfTokenKind::Content(_)));
687    }
688
689    #[test]
690    fn test_comment_with_leading_whitespace() {
691        let tokens = tokenize_gctf("  # indented comment");
692        assert!(
693            matches!(&tokens[0].kind, GctfTokenKind::Comment(t) if t == "  # indented comment")
694        );
695    }
696
697    #[test]
698    fn test_slash_slash_not_at_start_is_content() {
699        let tokens = tokenize_gctf("foo // bar");
700        assert!(matches!(&tokens[0].kind, GctfTokenKind::Content(t) if t == "foo // bar"));
701    }
702
703    #[test]
704    fn test_tab_only_line_is_blank() {
705        let tokens = tokenize_gctf("\t\t");
706        assert!(matches!(tokens[0].kind, GctfTokenKind::Blank));
707    }
708
709    // === tokenize_kv_line edge cases ===
710
711    #[test]
712    fn test_kv_line_empty_value() {
713        let (key, value) = tokenize_kv_line("key:").unwrap();
714        assert_eq!(key, "key");
715        assert_eq!(value, "");
716    }
717
718    #[test]
719    fn test_kv_line_colon_in_value() {
720        let (key, value) = tokenize_kv_line("url: http://host:8080").unwrap();
721        assert_eq!(key, "url");
722        assert_eq!(value, "http://host:8080");
723    }
724
725    #[test]
726    fn test_kv_line_value_with_spaces() {
727        let (key, value) = tokenize_kv_line("  cert  :  /path/to/cert.pem  ").unwrap();
728        assert_eq!(key, "cert");
729        assert_eq!(value, "/path/to/cert.pem");
730    }
731
732    #[test]
733    fn test_kv_line_tab_separator() {
734        let (key, value) = tokenize_kv_line("key\t:\tvalue").unwrap();
735        assert_eq!(key, "key");
736        assert_eq!(value, "value");
737    }
738
739    #[test]
740    fn test_kv_line_only_whitespace_key_produces_empty_key() {
741        let result = tokenize_kv_line("   : value");
742        assert!(result.is_some());
743        let (key, value) = result.unwrap();
744        assert_eq!(key, "");
745        assert_eq!(value, "value");
746    }
747
748    // === tokenize_extract_line edge cases ===
749
750    #[test]
751    fn test_extract_line_with_spaces() {
752        let (name, value) = tokenize_extract_line("  total  =  .response.total  ").unwrap();
753        assert_eq!(name, "total");
754        assert_eq!(value, ".response.total");
755    }
756
757    #[test]
758    fn test_extract_line_empty_value() {
759        let (name, value) = tokenize_extract_line("name=").unwrap();
760        assert_eq!(name, "name");
761        assert_eq!(value, "");
762    }
763
764    #[test]
765    fn test_extract_line_no_equals() {
766        assert_eq!(tokenize_extract_line(".just.a.path"), None);
767    }
768
769    #[test]
770    fn test_extract_line_whitespace_only() {
771        assert_eq!(tokenize_extract_line("   "), None);
772    }
773
774    #[test]
775    fn test_extract_line_only_whitespace_value() {
776        let (name, value) = tokenize_extract_line("name=   ").unwrap();
777        assert_eq!(name, "name");
778        assert_eq!(value, "");
779    }
780
781    // === tokenize_inline_options edge cases ===
782
783    #[test]
784    fn test_tokenize_options_empty() {
785        let opts = tokenize_inline_options("");
786        assert!(opts.is_empty());
787    }
788
789    #[test]
790    fn test_tokenize_options_only_spaces() {
791        let opts = tokenize_inline_options("   ");
792        assert!(opts.is_empty());
793    }
794
795    #[test]
796    fn test_tokenize_options_escaped_char() {
797        let opts = tokenize_inline_options(r#"key=va\"lue"#);
798        assert_eq!(opts.len(), 1);
799        assert_eq!(opts[0].0, "key");
800    }
801
802    #[test]
803    fn test_tokenize_options_single_quotes_not_special() {
804        let opts = tokenize_inline_options("key='hello world'");
805        assert_eq!(opts.len(), 2);
806        assert_eq!(opts[0], ("key".into(), "'hello".into()));
807    }
808
809    #[test]
810    fn test_tokenize_options_array_value() {
811        let opts = tokenize_inline_options(r#"redact=["field1","field2"]"#);
812        assert_eq!(opts.len(), 1);
813        assert_eq!(opts[0].0, "redact");
814        assert!(opts[0].1.contains("field1"));
815    }
816
817    #[test]
818    fn test_tokenize_options_multiple_spaces() {
819        let opts = tokenize_inline_options("  a=1   b=2  ");
820        assert_eq!(opts.len(), 2);
821        assert_eq!(opts[0], ("a".into(), "1".into()));
822        assert_eq!(opts[1], ("b".into(), "2".into()));
823    }
824
825    #[test]
826    fn test_tokenize_options_tab_separated() {
827        let opts = tokenize_inline_options("a=1\tb=2");
828        assert_eq!(opts.len(), 2);
829    }
830
831    #[test]
832    fn test_tokenize_options_quoted_key() {
833        let opts = tokenize_inline_options(r#""key"=value"#);
834        assert_eq!(opts.len(), 1);
835        assert_eq!(opts[0].0, "key");
836    }
837
838    #[test]
839    fn test_tokenize_options_empty_value() {
840        let opts = tokenize_inline_options("key=");
841        assert_eq!(opts.len(), 1);
842        assert_eq!(opts[0], ("key".into(), "".into()));
843    }
844
845    // === slice_str edge case (via other functions) ===
846
847    #[test]
848    fn test_span_tracking() {
849        let tokens = tokenize_gctf("# comment\n--- ENDPOINT ---\nhello");
850        assert_eq!(tokens[0].span, Span { start: 0, end: 9 });
851        assert_eq!(tokens[1].span, Span { start: 0, end: 16 });
852        assert_eq!(tokens[2].span, Span { start: 0, end: 5 });
853    }
854
855    #[test]
856    fn test_gctf_token_kind_equality() {
857        let t1 = GctfToken {
858            kind: GctfTokenKind::Blank,
859            line: 0,
860            span: Span { start: 0, end: 0 },
861        };
862        let t2 = GctfToken {
863            kind: GctfTokenKind::Blank,
864            line: 1,
865            span: Span { start: 0, end: 0 },
866        };
867        assert_ne!(t1, t2); // different line
868    }
869
870    #[test]
871    fn test_tokenize_attribute_block() {
872        let tokens = tokenize_gctf("#[timeout(30)]");
873        assert_eq!(tokens.len(), 1);
874        match &tokens[0].kind {
875            GctfTokenKind::AttributeBlock(content) => {
876                assert_eq!(content, "timeout(30)");
877            }
878            _ => panic!("expected AttributeBlock"),
879        }
880    }
881
882    #[test]
883    fn test_tokenize_attribute_block_with_spaces() {
884        let tokens = tokenize_gctf("#[  retry(3)  ]");
885        assert_eq!(tokens.len(), 1);
886        match &tokens[0].kind {
887            GctfTokenKind::AttributeBlock(content) => {
888                assert_eq!(content, "  retry(3)  ");
889            }
890            _ => panic!("expected AttributeBlock"),
891        }
892    }
893
894    #[test]
895    fn test_tokenize_attribute_block_flag() {
896        let tokens = tokenize_gctf("#[skip]");
897        assert_eq!(tokens.len(), 1);
898        match &tokens[0].kind {
899            GctfTokenKind::AttributeBlock(content) => {
900                assert_eq!(content, "skip");
901            }
902            _ => panic!("expected AttributeBlock"),
903        }
904    }
905
906    #[test]
907    fn test_tokenize_attribute_block_empty() {
908        let tokens = tokenize_gctf("#[]");
909        assert_eq!(tokens.len(), 1);
910        match &tokens[0].kind {
911            GctfTokenKind::AttributeBlock(content) => {
912                assert_eq!(content, "");
913            }
914            _ => panic!("expected AttributeBlock"),
915        }
916    }
917
918    #[test]
919    fn test_tokenize_attribute_block_quoted_value() {
920        let tokens = tokenize_gctf(r#"#[tag("smoke, slow")]"#);
921        assert_eq!(tokens.len(), 1);
922        match &tokens[0].kind {
923            GctfTokenKind::AttributeBlock(content) => {
924                assert_eq!(content, r#"tag("smoke, slow")"#);
925            }
926            _ => panic!("expected AttributeBlock"),
927        }
928    }
929
930    #[test]
931    fn test_tokenize_attribute_before_section() {
932        let tokens = tokenize_gctf("#[timeout(30)]\n--- REQUEST ---");
933        assert_eq!(tokens.len(), 2);
934        match &tokens[0].kind {
935            GctfTokenKind::AttributeBlock(content) => {
936                assert_eq!(content, "timeout(30)");
937            }
938            _ => panic!("expected AttributeBlock"),
939        }
940        match &tokens[1].kind {
941            GctfTokenKind::SectionHeader { name, .. } => {
942                assert_eq!(name, "REQUEST");
943            }
944            _ => panic!("expected SectionHeader"),
945        }
946    }
947
948    #[test]
949    fn test_tokenize_attribute_after_section() {
950        let tokens = tokenize_gctf("--- REQUEST ---\n{}\n#[skip]");
951        assert_eq!(tokens.len(), 3);
952        match &tokens[0].kind {
953            GctfTokenKind::SectionHeader { name, .. } => {
954                assert_eq!(name, "REQUEST");
955            }
956            _ => panic!("expected SectionHeader"),
957        }
958        match &tokens[2].kind {
959            GctfTokenKind::AttributeBlock(content) => {
960                assert_eq!(content, "skip");
961            }
962            _ => panic!("expected AttributeBlock"),
963        }
964    }
965}
apif_ast/gctf_tokenizer.rs

apif_ast/
gctf_tokenizer.rs
