liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

use std::{ops::Deref, sync::Arc};
use tokio::time::Duration;

use indicatif::{ProgressBar, ProgressStyle};

pub enum ProgressBarType {
    Counter,
    Bytes,
    None,
}

pub fn spinner_with_msg(msg: impl AsRef<str>) -> ProgressBar {
    let spinner = ProgressBar::new_spinner();
    spinner.set_message(msg.as_ref().to_owned());
    spinner.set_style(ProgressStyle::default_spinner());
    spinner.enable_steady_tick(Duration::from_millis(100));
    spinner
}

pub fn oxen_progress_bar(size: u64, progress_type: ProgressBarType) -> Arc<ProgressBar> {
    let bar = Arc::new(ProgressBar::new(size));
    bar.set_style(
        ProgressStyle::default_bar()
            .template(progress_type_to_template(progress_type).as_str())
            .unwrap()
            .progress_chars("🌾🐂➖"),
    );
    bar
}

pub fn oxen_progress_bar_with_msg(size: u64, msg: impl AsRef<str>) -> Arc<ProgressBar> {
    let bar = Arc::new(ProgressBar::new(size));
    bar.set_message(msg.as_ref().to_owned());
    bar.set_style(
        ProgressStyle::default_bar()
            .template(progress_type_to_template(ProgressBarType::Counter).as_str())
            .unwrap()
            .progress_chars("🌾🐂➖"),
    );
    bar
}

// Modify styling to oxen bar - necessary for bars which start out as spinners
pub fn oxify_bar(bar: Arc<ProgressBar>, progress_type: ProgressBarType) -> Arc<ProgressBar> {
    bar.set_style(
        ProgressStyle::default_bar()
            .template(progress_type_to_template(progress_type).as_str())
            .unwrap()
            .progress_chars("🌾🐂➖"),
    );
    bar
}

pub fn progress_type_to_template(progress_type: ProgressBarType) -> String {
    match progress_type {
        ProgressBarType::Counter => {
            "{spinner:.green} {msg} [{elapsed_precise}] [{wide_bar}] {pos}/{len}".to_string()
        }
        ProgressBarType::Bytes => {
            "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {bytes}/{total_bytes}".to_string()
        }
        ProgressBarType::None => "{spinner:.green} [{elapsed_precise}] [{wide_bar}]".to_string(),
    }
}

/// Wraps a [`ProgressBar`] with a `Drop` impl that calls the `finish_and_clear` method.
/// This is so that the progress bar is always finished even if it's used in a context
/// where a function early-returns e.g. on an error.
pub(crate) struct FinishOnDropProgressBar(pub ProgressBar);

impl Drop for FinishOnDropProgressBar {
    fn drop(&mut self) {
        self.0.finish_and_clear();
    }
}

impl Deref for FinishOnDropProgressBar {
    type Target = ProgressBar;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}