liendl_tokenizer 0.1.0

A simple BPE tokenizer for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

use std::{fs::File, io::{self, BufRead, BufReader, Lines}, marker::PhantomData};

use crate::shared::error::LoaderError;

pub struct Closed;
pub struct Open;

pub enum LoaderOptions {
    TXT
}

pub struct TextLoader<S> {
    option: LoaderOptions,
    state: PhantomData<S>,
    path: String,
    lines: Option<Lines<BufReader<File>>>
}

impl<S> TextLoader<S> {
    pub fn new(option: LoaderOptions, path: &str) -> Self {
        TextLoader {
            option,
            state: PhantomData,
            path: path.to_string(),
            lines: None,
        }
    }
}

impl TextLoader<Open> {
    pub fn read_line(&mut self) -> Option<io::Result<String>> {
        self.lines.as_mut()?.next()
    }

    pub fn close(self) -> TextLoader<Closed> {
        TextLoader::new(self.option, self.path.as_str())
    }
}

impl TextLoader<Closed> {
    pub fn open(self) -> Result<TextLoader<Open>, LoaderError> {
        let file = match File::open(&self.path) {
            Ok(f) => f,
            Err(e) => {
                return Err(LoaderError::new(format!("Unable to open file: {}", e).as_str()));
            },
        };

        let buffer = BufReader::new(file);

        Ok(TextLoader {
            option: self.option,
            path: self.path,
            state: PhantomData,
            lines: Some(buffer.lines()),
        })
    }
}