rust_tokenizers 3.1.3

High performance tokenizers for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

#[cfg(feature = "proto-compile")]
use std::fs;

fn main() {
    #[cfg(feature = "proto-compile")]
        {
            let out_path = "src/preprocessing/vocab/sentencepiece_proto";
            let out_file_name = "src/preprocessing/vocab/sentencepiece_proto/sentencepiece_model.proto";
            let proto_path = "protos/sentencepiece_model.proto";

            let metadata = fs::metadata(out_file_name);

            if !(metadata.is_ok()) {
                protobuf_codegen_pure::Codegen::new()
                    .out_dir(out_path)
                    .inputs(&[proto_path])
                    .include("protos")
                    .run()
                    .expect("Codegen failed.");
            }
        }
}