xan 0.55.0

The CSV magician
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112

use std::io::{self, Cursor, Read};

use regex::bytes::Regex;

pub fn consume_cdx_header<R: Read>(reader: &mut R) -> io::Result<bool> {
    let mut buf = [0u8; 5];

    reader.read_exact(&mut buf)?;

    Ok(&buf == b" CDX ")
}

type RecombobulatedReader<R> = io::Chain<Cursor<Vec<u8>>, R>;

pub fn consume_header_until<R: Read>(
    reader: R,
    pattern: &Regex,
) -> io::Result<Option<(u64, RecombobulatedReader<R>)>> {
    let mut line_reader = simd_csv::LineReader::from_reader(reader);
    let mut pos = line_reader.position();
    let mut header_opt: Option<Vec<u8>> = None;

    while let Some(line) = line_reader.read_line()? {
        if !pattern.is_match(line) {
            pos = line_reader.position();
            continue;
        }

        header_opt = Some(line.to_vec());

        break;
    }

    if let Some(mut fixed_data) = header_opt {
        let bufreader = line_reader.into_bufreader();
        fixed_data.push(b'\n');
        fixed_data.extend(bufreader.buffer());

        let fixed_reader = Cursor::new(fixed_data).chain(bufreader.into_inner());

        Ok(Some((pos, fixed_reader)))
    } else {
        Ok(None)
    }
}

pub fn consume_header_while<R: Read>(
    reader: R,
    pattern: &Regex,
) -> io::Result<Option<(u64, RecombobulatedReader<R>)>> {
    let mut line_reader = simd_csv::LineReader::from_reader(reader);
    let mut pos = line_reader.position();
    let mut header_opt: Option<Vec<u8>> = None;

    while let Some(line) = line_reader.read_line()? {
        if pattern.is_match(line) {
            pos = line_reader.position();
            continue;
        }

        header_opt = Some(line.to_vec());

        break;
    }

    if let Some(mut fixed_data) = header_opt {
        let bufreader = line_reader.into_bufreader();
        fixed_data.push(b'\n');
        fixed_data.extend(bufreader.buffer());

        let fixed_reader = Cursor::new(fixed_data).chain(bufreader.into_inner());

        Ok(Some((pos, fixed_reader)))
    } else {
        Ok(None)
    }
}

pub fn consume_lines<R: Read>(
    reader: R,
    limit: usize,
) -> io::Result<Option<(u64, RecombobulatedReader<R>)>> {
    let mut line_reader = simd_csv::LineReader::from_reader(reader);
    let mut pos = line_reader.position();
    let mut seen: usize = 0;
    let mut header_opt: Option<Vec<u8>> = None;

    while let Some(line) = line_reader.read_line()? {
        seen += 1;

        if seen <= limit {
            pos = line_reader.position();
            continue;
        }

        header_opt = Some(line.to_vec());

        break;
    }

    if let Some(mut fixed_data) = header_opt {
        let bufreader = line_reader.into_bufreader();
        fixed_data.push(b'\n');
        fixed_data.extend(bufreader.buffer());

        let fixed_reader = Cursor::new(fixed_data).chain(bufreader.into_inner());

        Ok(Some((pos, fixed_reader)))
    } else {
        Ok(None)
    }
}