cpd_tokenizer/
embedded.rs

1pub fn blank_ranges_preserve_newlines(source: &str, ranges: &[[usize; 2]]) -> String {
2    let mut src_bytes = source.as_bytes().to_vec();
3    let mut sorted: Vec<[usize; 2]> = ranges.to_vec();
4    sorted.sort_by_key(|r| r[0]);
5    for i in 1..sorted.len() {
6        if sorted[i][0] < sorted[i - 1][1] {
7            panic!(
8                "overlapping ranges detected: [{}, {}) overlaps [{}, {})",
9                sorted[i - 1][0],
10                sorted[i - 1][1],
11                sorted[i][0],
12                sorted[i][1],
13            );
14        }
15    }
16    sorted.sort_by(|a, b| b[0].cmp(&a[0]));
17    for &[start, end] in &sorted {
18        for byte in &mut src_bytes[start..end] {
19            match byte {
20                b'\n' => {}
21                _ => *byte = b' ',
22            }
23        }
24    }
25    String::from_utf8(src_bytes).expect("blanking preserves UTF-8 validity")
26}
27
28#[cfg(test)]
29mod tests {
30    use super::*;
31
32    #[test]
33    fn empty_ranges_returns_source_unchanged() {
34        let s = "hello\nworld";
35        let result = blank_ranges_preserve_newlines(s, &[]);
36        assert_eq!(result, s);
37    }
38
39    #[test]
40    fn single_range_blanks_non_newlines() {
41        let s = "ab\ncd";
42        let result = blank_ranges_preserve_newlines(s, &[[0, 5]]);
43        assert_eq!(result, "  \n  ");
44    }
45
46    #[test]
47    #[should_panic(expected = "overlapping ranges")]
48    fn overlapping_ranges_panics() {
49        let s = "hello";
50        let _ = blank_ranges_preserve_newlines(s, &[[0, 3], [2, 5]]);
51    }
52
53    #[test]
54    fn multiple_ranges_all_blanked() {
55        let s = "foo\nbar\nbaz";
56        let result = blank_ranges_preserve_newlines(s, &[[0, 3], [4, 7]]);
57        assert_eq!(result, "   \n   \nbaz");
58    }
59
60    #[test]
61    fn right_to_left_no_index_drift() {
62        let s = "abcdefghij";
63        let result = blank_ranges_preserve_newlines(s, &[[0, 3], [6, 9]]);
64        assert_eq!(result, "   def   j");
65    }
66}
cpd_tokenizer/embedded.rs

cpd_tokenizer/
embedded.rs
