rust_tokenizers 7.0.2

High performance tokenizers for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

#[cfg(feature = "proto-compile")]
use std::fs;

fn main() {
    #[cfg(feature = "proto-compile")]
    {
        let out_path = "src/vocab/sentencepiece_proto";
        let out_file_name = "src/vocab/sentencepiece_proto/sentencepiece_model.proto";
        let proto_path = "protos/sentencepiece_model.proto";

        let metadata = fs::metadata(out_file_name);

        if metadata.is_err() {
            protobuf_codegen_pure::Codegen::new()
                .out_dir(out_path)
                .inputs(&[proto_path])
                .include("protos")
                .run()
                .expect("Codegen failed.");
        }
    }
}