overbrainer 0.10.0

Distill a large LLM into a smaller open-weights model: generate questions, collect answers, fine-tune, visualize.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

use std::io;
use std::path::PathBuf;

/// Errors of a compare: its files, its eval set, its judge.
#[derive(Debug, thiserror::Error)]
pub enum CompareError {
    /// A file or directory could not be read or written.
    #[error("cannot access {}", path.display())]
    Io {
        /// The file or directory.
        path: PathBuf,
        /// Underlying I/O error.
        #[source]
        source: io::Error,
    },
    /// A JSON file of the compare does not parse.
    #[error("{} is not valid", path.display())]
    Json {
        /// The file.
        path: PathBuf,
        /// Underlying parse error.
        #[source]
        source: serde_json::Error,
    },
    /// `data/eval.jsonl` cannot be read.
    #[error(transparent)]
    Dataset(#[from] crate::dataset::DatasetError),
    /// The judge prompt cannot be rendered.
    #[error(transparent)]
    Prompt(#[from] crate::prompts::PromptError),
    /// The judge failed on a question, retries spent.
    #[error("the judge failed on question {id}")]
    Judge {
        /// The question.
        id: String,
        /// Underlying error.
        #[source]
        source: crate::llm::LlmError,
    },
    /// The eval set has no question to ask.
    #[error("{} has no question with an answer: run `overbrainer split` first", .0.display())]
    NoQuestions(PathBuf),
    /// The GGUF on the target is not named by an absolute path.
    #[error("the GGUF on the target must be an absolute path, not {0}")]
    RelativeModel(String),
    /// No compare has this ID.
    #[error("no compare {0} in runs/")]
    NotFound(String),
}

/// Maps an I/O error on `path` to [`CompareError::Io`].
pub(crate) fn io_error(path: &std::path::Path) -> impl FnOnce(io::Error) -> CompareError + '_ {
    move |source| CompareError::Io {
        path: path.to_path_buf(),
        source,
    }
}

/// Maps a JSON error on `path` to [`CompareError::Json`].
pub(crate) fn json_error(
    path: &std::path::Path,
) -> impl FnOnce(serde_json::Error) -> CompareError + '_ {
    move |source| CompareError::Json {
        path: path.to_path_buf(),
        source,
    }
}