synthclaw 0.1.3

Lightweight synthetic data generation library/CLI.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

pub mod hf;
mod local;

pub use hf::{HuggingFaceSource, Split};
pub use local::LocalSource;

use crate::Result;
use serde_json::Value;

pub struct Record {
    pub data: Value,
    pub index: usize,
}

pub trait DataSource: Send + Sync {
    fn info(&self) -> &DatasetInfo;
    fn load(&mut self, sample: Option<usize>) -> Result<Vec<Record>>;
}

#[derive(Default)]
pub struct DatasetInfo {
    pub name: String,
    pub description: Option<String>,
    pub num_rows: usize,
    pub columns: Vec<String>,
    pub splits: Vec<Split>,
}