corgi-rs 0.2.2

Vehicle VIN decoding library based on NHTSA database for Rust
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

use std::io::{BufWriter, Write};
use std::str::FromStr;
use std::{fs::File, path::Path};

use fst::MapBuilder;

use build_shared::{Lookup, UntilNextKey};
use rkyv::rancor::Error;

use crate::build_shared::{Make, RkyvSerialize, Saveable, SchemaId};

#[path = "src/build_shared.rs"]
mod build_shared;

fn main() {
    let out_dir = dirs::home_dir()
        .expect("HOME env variable not set")
        .join(".corgi-rs-cache");
    std::fs::create_dir_all(&out_dir).expect("dir create");

    let lookup_fst_path = Path::new(&out_dir).join(format!("{}.fst", Lookup::base_file_name()));
    let lookup_values_path = Path::new(&out_dir).join(format!("{}.bin", Lookup::base_file_name()));
    let lookup_task_handle = if !lookup_fst_path.exists() || !lookup_values_path.exists() {
        let fst_file = File::create(&lookup_fst_path).expect("fst file create");
        let values_file = File::create(&lookup_values_path).expect("values file create");

        let handle = std::thread::spawn(|| {
            generate_fst_map::<Lookup>(
                &Path::new(&format!("assets/{}.csv", Lookup::base_file_name())),
                fst_file,
                values_file,
            )
            .expect("generate fst lookup");
        });

        Some(handle)
    } else {
        None
    };

    let make_fst_path = Path::new(&out_dir).join(format!("{}.fst", Make::base_file_name()));
    let make_values_path = Path::new(&out_dir).join(format!("{}.bin", Make::base_file_name()));
    let make_task_handle = if !make_fst_path.exists() || !make_values_path.exists() {
        let fst_file = File::create(&make_fst_path).expect("fst file create");
        let values_file = File::create(&make_values_path).expect("values file create");

        let handle = std::thread::spawn(|| {
            generate_fst_map::<Make>(
                &Path::new(&format!("assets/{}.csv", Make::base_file_name())),
                fst_file,
                values_file,
            )
            .expect("generate fst make")
        });

        Some(handle)
    } else {
        None
    };

    let schema_fst_path = Path::new(&out_dir).join(format!("{}.fst", SchemaId::base_file_name()));
    let schema_values_path =
        Path::new(&out_dir).join(format!("{}.bin", SchemaId::base_file_name()));
    let schema_task_handle = if !schema_fst_path.exists() || !schema_values_path.exists() {
        let fst_file = File::create(&schema_fst_path).expect("fst file create");
        let values_file = File::create(&schema_values_path).expect("values file create");

        let handle = std::thread::spawn(|| {
            generate_fst_map::<SchemaId>(
                &Path::new(&format!("assets/{}.csv", SchemaId::base_file_name())),
                fst_file,
                values_file,
            )
            .expect("generate fst schema")
        });

        Some(handle)
    } else {
        None
    };

    lookup_task_handle.map(|h| h.join().expect("lookup task join"));
    make_task_handle.map(|h| h.join().expect("make task join"));
    schema_task_handle.map(|h| h.join().expect("schema task join"));

    println!("cargo:rerun-if-changed=src/build_shared.rs");
}

fn generate_fst_map<V>(
    csv_path: &Path,
    fst_file: File,
    values_file: File,
) -> Result<(), Box<dyn std::error::Error>>
where
    V: RkyvSerialize + FromStr,
{
    let csv = std::fs::read_to_string(csv_path)?;

    let fst_writer = BufWriter::new(fst_file);
    let mut fst_builder = MapBuilder::new(fst_writer)?;

    let mut values_writer = BufWriter::new(values_file);

    let mut current_offset = 0 as u64;

    // Skip header line
    let mut csv_lines = csv.lines().skip(1).peekable();
    while let Some((key, values)) = csv_lines.next_key() {
        let values = values
            .into_iter()
            .map(|v| {
                V::from_str(v)
                    .map_err(|_| std::io::Error::other("value from str"))
                    .expect("str parse")
            })
            .collect::<Vec<_>>();
        let bytes = rkyv::to_bytes::<Error>(&values)?;
        values_writer.write_all(&bytes)?;

        let offset_len_combined = (current_offset << 32) | bytes.len() as u64;
        fst_builder.insert(key, offset_len_combined)?;

        current_offset += bytes.len() as u64;
    }

    fst_builder.finish()?;
    values_writer.flush()?;

    Ok(())
}