hpt-dataloader 0.1.3

An internal library for data loading for hpt
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

mod compression_trait;
pub mod data_loader;
pub use compression_trait::CompressionAlgo;
pub use compression_trait::DataLoaderTrait;
pub use compression_trait::Meta;
pub use compression_trait::{DataLoader, TensorLoader, TensorSaver};
pub use data_loader::Endian;
pub use flate2::write::{DeflateEncoder, GzEncoder, ZlibEncoder};
pub use from_safetensors::from_safetensors::FromSafeTensors;
pub use struct_save::gen_header;
pub use struct_save::load::{Load, MetaLoad};
pub use struct_save::save::save;
pub use struct_save::save::Save;
pub use utils::CPUTensorCreator;
mod struct_save {
    pub mod gen_header;
    pub mod load;
    pub mod save;
}

pub mod from_safetensors {
    pub mod from_safetensors;
}

pub mod load;
pub mod save;
pub mod utils;

pub(crate) const CHUNK_BUFF: usize = 1024 * 1024;