brainwires-datasets 0.7.0

Training data pipelines for the Brainwires Agent Framework — JSONL I/O, tokenization, deduplication, format conversion
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use thiserror::Error;

/// Errors that can occur during dataset operations.
#[derive(Error, Debug)]
pub enum DatasetError {
    /// I/O error reading or writing data.
    #[error("I/O error: {0}")]
    Io(#[from] std::io::Error),

    /// JSON serialization or deserialization error.
    #[error("JSON parse error: {0}")]
    Json(#[from] serde_json::Error),

    /// Dataset validation failed.
    #[error("Validation error: {message}")]
    Validation {
        /// Description of the validation failure.
        message: String,
    },

    /// Format conversion error between providers.
    #[error("Format conversion error: {message}")]
    FormatConversion {
        /// Description of the conversion failure.
        message: String,
    },

    /// Tokenizer encoding or decoding error.
    #[error("Tokenizer error: {message}")]
    Tokenizer {
        /// Description of the tokenizer failure.
        message: String,
    },

    /// Index is out of bounds for the dataset.
    #[error("Index out of bounds: {index} (len: {len})")]
    IndexOutOfBounds {
        /// The requested index.
        index: usize,
        /// The actual length of the dataset.
        len: usize,
    },

    /// The dataset is empty.
    #[error("Empty dataset")]
    EmptyDataset,

    /// Generic error with a message.
    #[error("{0}")]
    Other(
        /// Error message.
        String,
    ),
}

/// Convenience alias for `Result<T, DatasetError>`.
pub type DatasetResult<T> = Result<T, DatasetError>;