lean_ctx_compress/
csv.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! CSV/TSV → record text + row-group chunks (EPIC 12.13).
4//!
5//! A small RFC-4180-aware parser (quoted fields, escaped `""`, embedded
6//! delimiters/newlines) turns tabular data into `header: value` records so an
7//! LLM sees labeled fields, not bare columns. Chunks are row groups, each
8//! prefixed with the header for standalone context. Degrades gracefully: any
9//! non-empty input yields at least one non-empty chunk.
10
11/// Rows per chunk — keeps each chunk small while preserving header context.
12const ROWS_PER_CHUNK: usize = 20;
13
14/// Parse delimited text into rows of fields (RFC-4180 quoting aware).
15#[must_use]
16pub fn parse(input: &str, delimiter: char) -> Vec<Vec<String>> {
17    let mut rows = Vec::new();
18    let mut field = String::new();
19    let mut record: Vec<String> = Vec::new();
20    let mut in_quotes = false;
21    let mut chars = input.chars().peekable();
22
23    while let Some(ch) = chars.next() {
24        if in_quotes {
25            if ch == '"' {
26                if chars.peek() == Some(&'"') {
27                    field.push('"');
28                    chars.next();
29                } else {
30                    in_quotes = false;
31                }
32            } else {
33                field.push(ch);
34            }
35        } else if ch == '"' {
36            in_quotes = true;
37        } else if ch == delimiter {
38            record.push(std::mem::take(&mut field));
39        } else if ch == '\n' || ch == '\r' {
40            // Swallow a CRLF pair as one terminator.
41            if ch == '\r' && chars.peek() == Some(&'\n') {
42                chars.next();
43            }
44            record.push(std::mem::take(&mut field));
45            push_record(&mut rows, std::mem::take(&mut record));
46        } else {
47            field.push(ch);
48        }
49    }
50    // Trailing field/record (no final newline).
51    if !field.is_empty() || !record.is_empty() {
52        record.push(field);
53        push_record(&mut rows, record);
54    }
55    rows
56}
57
58/// Append a record unless it is entirely empty (skips blank lines).
59fn push_record(rows: &mut Vec<Vec<String>>, record: Vec<String>) {
60    if record.iter().any(|f| !f.trim().is_empty()) {
61        rows.push(record);
62    }
63}
64
65/// Render all data rows as `header: value | …` records.
66#[must_use]
67pub fn to_text(input: &str, delimiter: char) -> String {
68    let rows = parse(input, delimiter);
69    record_lines(&rows).join("\n")
70}
71
72/// Row-group chunks; each chunk repeats the header line for context.
73#[must_use]
74pub fn chunks(input: &str, delimiter: char) -> Vec<String> {
75    let rows = parse(input, delimiter);
76    if rows.is_empty() {
77        return Vec::new();
78    }
79    let lines = record_lines(&rows);
80    if lines.is_empty() {
81        // Header-only (no data rows): the header itself is the content.
82        return vec![rows[0].join(&delimiter.to_string())];
83    }
84    let header_line = rows[0].join(&delimiter.to_string());
85    lines
86        .chunks(ROWS_PER_CHUNK)
87        .map(|group| format!("{}\n{}", header_line, group.join("\n")))
88        .collect()
89}
90
91/// Map data rows (everything after row 0) to `col: val | col: val` lines.
92fn record_lines(rows: &[Vec<String>]) -> Vec<String> {
93    if rows.len() < 2 {
94        return Vec::new();
95    }
96    let header = &rows[0];
97    rows[1..]
98        .iter()
99        .map(|row| {
100            row.iter()
101                .enumerate()
102                .map(|(i, val)| {
103                    let col = header
104                        .get(i)
105                        .map_or_else(|| format!("col{i}"), |h| h.trim().to_string());
106                    format!("{}: {}", col, val.trim())
107                })
108                .collect::<Vec<_>>()
109                .join(" | ")
110        })
111        .filter(|l| !l.trim().is_empty())
112        .collect()
113}
114
115#[cfg(test)]
116pub mod tests {
117    use super::*;
118
119    #[test]
120    fn parses_quoted_fields_with_embedded_delimiter() {
121        let rows = parse("a,b\n\"x,y\",\"he said \"\"hi\"\"\"\n", ',');
122        assert_eq!(rows.len(), 2);
123        assert_eq!(rows[1][0], "x,y");
124        assert_eq!(rows[1][1], "he said \"hi\"");
125    }
126
127    #[test]
128    fn renders_labeled_records() {
129        let text = to_text("name,age\nAlice,30\nBob,25", ',');
130        assert!(text.contains("name: Alice | age: 30"));
131        assert!(text.contains("name: Bob | age: 25"));
132    }
133
134    #[test]
135    fn chunks_repeat_header() {
136        let c = chunks("h1,h2\na,b\nc,d", ',');
137        assert_eq!(c.len(), 1);
138        assert!(c[0].starts_with("h1,h2\n"));
139    }
140
141    #[test]
142    fn header_only_input_still_chunks() {
143        let c = chunks("just,a,header", ',');
144        assert_eq!(c.len(), 1);
145        assert_eq!(c[0], "just,a,header");
146    }
147
148    #[test]
149    fn skips_blank_lines() {
150        let rows = parse("a\n\n\nb", ',');
151        assert_eq!(rows.len(), 2);
152    }
153}
lean_ctx_compress/csv.rs

lean_ctx_compress/
csv.rs
