scrubcsv 1.0.0

Remove bad lines from large CSV files and normalize the rest
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

//! Make unique identifiers from messy strings.
//!
//! Imported from `dbcrossbar`, another open source Faraday project with
//! the same copyright holder and license.

use std::collections::HashSet;

use super::ColumnNameCleaner;
use crate::{format_err, Result};

/// Turns arbitrary Unicode names into unique, lowercase ASCII identifiers. All
/// identifiers start with an underscore or a lowercase ASCII letter, followed
/// by zero or more underscores, lowercase ASCII letters and digits.
#[derive(Debug, Default)]
pub(crate) struct Uniquifier {
    /// Identifiers that we have already generated.
    used: HashSet<String>,
}

impl ColumnNameCleaner for Uniquifier {
    fn unique_id_for(&mut self, name: &str) -> Result<String> {
        let id = name_to_lowercase_id(name);
        if self.used.insert(id.to_owned()) {
            Ok(id)
        } else {
            let mut offset = 1;
            while offset < 50 {
                offset += 1;
                let alt_id = format!("{}_{}", id, offset);
                if self.used.insert(alt_id.to_owned()) {
                    return Ok(alt_id);
                }
            }
            Err(format_err!("too many column name collisions"))
        }
    }
}

#[test]
fn uniquifier_generates_unique_ids() {
    let examples = &[
        ("a", "a"),
        ("A", "a_2"),
        ("a_2", "a_2_2"), // Sneaky.
        ("B", "b"),
    ];
    let mut uniqifier = Uniquifier::default();
    for &(input, expected) in examples {
        assert_eq!(uniqifier.unique_id_for(input).unwrap(), expected);
    }
}

/// Given a unique string, turn it into a lower-case identifier.
fn name_to_lowercase_id(name: &str) -> String {
    if name.is_empty() {
        "_".to_owned()
    } else {
        name.char_indices()
            .map(|(idx, c)| {
                if c == '_' || c.is_ascii_lowercase() {
                    c
                } else if c.is_ascii_uppercase() {
                    c.to_ascii_lowercase()
                } else if idx != 0 && c.is_ascii_digit() {
                    c
                } else {
                    '_'
                }
            })
            .collect::<String>()
    }
}

#[test]
fn name_to_lowercase_id_cleans_non_id_characters() {
    let examples = &[("", "_"), ("_aA1?", "_aa1_"), ("1", "_")];
    for &(input, expected) in examples {
        assert_eq!(name_to_lowercase_id(input), expected);
    }
}