lean_ctx/core/extractors/
csv.rs

1//! CSV/TSV → record text + row-group chunks (EPIC 12.13).
2//!
3//! A small RFC-4180-aware parser (quoted fields, escaped `""`, embedded
4//! delimiters/newlines) turns tabular data into `header: value` records so an
5//! LLM sees labeled fields, not bare columns. Chunks are row groups, each
6//! prefixed with the header for standalone context. Degrades gracefully: any
7//! non-empty input yields at least one non-empty chunk.
8
9/// Rows per chunk — keeps each chunk small while preserving header context.
10const ROWS_PER_CHUNK: usize = 20;
11
12/// Parse delimited text into rows of fields (RFC-4180 quoting aware).
13#[must_use]
14pub fn parse(input: &str, delimiter: char) -> Vec<Vec<String>> {
15    let mut rows = Vec::new();
16    let mut field = String::new();
17    let mut record: Vec<String> = Vec::new();
18    let mut in_quotes = false;
19    let mut chars = input.chars().peekable();
20
21    while let Some(ch) = chars.next() {
22        if in_quotes {
23            if ch == '"' {
24                if chars.peek() == Some(&'"') {
25                    field.push('"');
26                    chars.next();
27                } else {
28                    in_quotes = false;
29                }
30            } else {
31                field.push(ch);
32            }
33        } else if ch == '"' {
34            in_quotes = true;
35        } else if ch == delimiter {
36            record.push(std::mem::take(&mut field));
37        } else if ch == '\n' || ch == '\r' {
38            // Swallow a CRLF pair as one terminator.
39            if ch == '\r' && chars.peek() == Some(&'\n') {
40                chars.next();
41            }
42            record.push(std::mem::take(&mut field));
43            push_record(&mut rows, std::mem::take(&mut record));
44        } else {
45            field.push(ch);
46        }
47    }
48    // Trailing field/record (no final newline).
49    if !field.is_empty() || !record.is_empty() {
50        record.push(field);
51        push_record(&mut rows, record);
52    }
53    rows
54}
55
56/// Append a record unless it is entirely empty (skips blank lines).
57fn push_record(rows: &mut Vec<Vec<String>>, record: Vec<String>) {
58    if record.iter().any(|f| !f.trim().is_empty()) {
59        rows.push(record);
60    }
61}
62
63/// Render all data rows as `header: value | …` records.
64#[must_use]
65pub fn to_text(input: &str, delimiter: char) -> String {
66    let rows = parse(input, delimiter);
67    record_lines(&rows).join("\n")
68}
69
70/// Row-group chunks; each chunk repeats the header line for context.
71#[must_use]
72pub fn chunks(input: &str, delimiter: char) -> Vec<String> {
73    let rows = parse(input, delimiter);
74    if rows.is_empty() {
75        return Vec::new();
76    }
77    let lines = record_lines(&rows);
78    if lines.is_empty() {
79        // Header-only (no data rows): the header itself is the content.
80        return vec![rows[0].join(&delimiter.to_string())];
81    }
82    let header_line = rows[0].join(&delimiter.to_string());
83    lines
84        .chunks(ROWS_PER_CHUNK)
85        .map(|group| format!("{}\n{}", header_line, group.join("\n")))
86        .collect()
87}
88
89/// Map data rows (everything after row 0) to `col: val | col: val` lines.
90fn record_lines(rows: &[Vec<String>]) -> Vec<String> {
91    if rows.len() < 2 {
92        return Vec::new();
93    }
94    let header = &rows[0];
95    rows[1..]
96        .iter()
97        .map(|row| {
98            row.iter()
99                .enumerate()
100                .map(|(i, val)| {
101                    let col = header
102                        .get(i)
103                        .map_or_else(|| format!("col{i}"), |h| h.trim().to_string());
104                    format!("{}: {}", col, val.trim())
105                })
106                .collect::<Vec<_>>()
107                .join(" | ")
108        })
109        .filter(|l| !l.trim().is_empty())
110        .collect()
111}
112
113#[cfg(test)]
114mod tests {
115    use super::*;
116
117    #[test]
118    fn parses_quoted_fields_with_embedded_delimiter() {
119        let rows = parse("a,b\n\"x,y\",\"he said \"\"hi\"\"\"\n", ',');
120        assert_eq!(rows.len(), 2);
121        assert_eq!(rows[1][0], "x,y");
122        assert_eq!(rows[1][1], "he said \"hi\"");
123    }
124
125    #[test]
126    fn renders_labeled_records() {
127        let text = to_text("name,age\nAlice,30\nBob,25", ',');
128        assert!(text.contains("name: Alice | age: 30"));
129        assert!(text.contains("name: Bob | age: 25"));
130    }
131
132    #[test]
133    fn chunks_repeat_header() {
134        let c = chunks("h1,h2\na,b\nc,d", ',');
135        assert_eq!(c.len(), 1);
136        assert!(c[0].starts_with("h1,h2\n"));
137    }
138
139    #[test]
140    fn header_only_input_still_chunks() {
141        let c = chunks("just,a,header", ',');
142        assert_eq!(c.len(), 1);
143        assert_eq!(c[0], "just,a,header");
144    }
145
146    #[test]
147    fn skips_blank_lines() {
148        let rows = parse("a\n\n\nb", ',');
149        assert_eq!(rows.len(), 2);
150    }
151}
lean_ctx/core/extractors/csv.rs

lean_ctx/core/extractors/
csv.rs
