liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

use crate::error::OxenError;
use crate::model::diff::change_type::ChangeType;
use crate::model::diff::text_diff::LineDiff;
use crate::model::diff::text_diff::TextDiff;

use difference::{Changeset, Difference};
use std::path::PathBuf;

/// Adds a slice of lines from a text block to the result vector with a given modification type.
fn add_lines_to_diff(
    result: &mut TextDiff,
    text_block: &str,
    modification: ChangeType,
    lines_to_take: Option<(usize, usize)>,
) {
    let lines: Vec<&str> = text_block.split('\n').collect();
    let (start, end) = lines_to_take.unwrap_or((0, lines.len()));

    // Ensure start and end are within bounds
    let start = start.min(lines.len());
    let end = end.min(lines.len());

    if start >= end {
        return;
    }

    for line in &lines[start..end] {
        result.lines.push(LineDiff {
            modification,
            text: line.to_string(),
        });
    }
}

pub fn diff(
    original_data: Option<String>,
    version_file_1: Option<PathBuf>,
    compare_data: Option<String>,
    version_file_2: Option<PathBuf>,
) -> Result<TextDiff, OxenError> {
    let mut result = TextDiff {
        filename1: version_file_1
            .clone()
            .map(|p| p.to_string_lossy().to_string()),
        filename2: version_file_2
            .clone()
            .map(|p| p.to_string_lossy().to_string()),
        ..Default::default()
    };

    let original_data = original_data.unwrap_or_default();
    let compare_data = compare_data.unwrap_or_default();

    let Changeset { diffs, .. } = Changeset::new(&original_data, &compare_data, "\n");
    log::debug!("Changeset created with {} diffs", diffs.len());

    // Find the indices of all Add or Rem changes
    let change_indices: Vec<usize> = diffs
        .iter()
        .enumerate()
        .filter(|(_, d)| !matches!(d, Difference::Same(_)))
        .map(|(i, _)| i)
        .collect();

    // If there are no changes, return an empty diff
    if change_indices.is_empty() {
        log::debug!("No changes detected, returning empty TextDiff.");
        return Ok(result);
    }

    let mut last_processed_diff_idx: i32 = -1;
    let mut post_context_lines_from_prev_chunk = 0;
    let mut is_first_chunk = true;

    for &change_idx in &change_indices {
        if (change_idx as i32) <= last_processed_diff_idx {
            continue;
        }
        log::debug!("Processing change at index: {change_idx}");

        if !is_first_chunk {
            result.lines.push(LineDiff {
                modification: ChangeType::Unchanged,
                text: "...".to_string(),
            });
        }
        is_first_chunk = false;

        let context_diff_idx = change_idx.saturating_sub(1);
        let mut pre_context_lines_to_skip = 0;

        if (context_diff_idx as i32) == last_processed_diff_idx {
            pre_context_lines_to_skip = post_context_lines_from_prev_chunk;
        }

        if change_idx > 0
            && let Some(Difference::Same(text)) = diffs.get(context_diff_idx)
        {
            let lines: Vec<_> = text.split('\n').collect();
            let desired_start = lines.len().saturating_sub(3);
            let actual_start = desired_start.max(pre_context_lines_to_skip);
            log::debug!(
                "Adding pre-context from diff [{context_diff_idx}], lines [{actual_start}..]"
            );
            add_lines_to_diff(
                &mut result,
                text,
                ChangeType::Unchanged,
                Some((actual_start, lines.len())),
            );
        }
        post_context_lines_from_prev_chunk = 0;

        let mut current_idx = change_idx;
        while let Some(diff) = diffs.get(current_idx) {
            match diff {
                Difference::Add(text) => {
                    log::debug!("Adding Added block at index {current_idx}");
                    add_lines_to_diff(&mut result, text, ChangeType::Added, None);
                }
                Difference::Rem(text) => {
                    log::debug!("Adding Removed block at index {current_idx}");
                    add_lines_to_diff(&mut result, text, ChangeType::Removed, None);
                }
                Difference::Same(_) => {
                    break;
                }
            }
            last_processed_diff_idx = current_idx as i32;
            current_idx += 1;
        }

        if let Some(Difference::Same(text)) = diffs.get(current_idx) {
            let lines: Vec<_> = text.split('\n').collect();
            let count = 2.min(lines.len());
            log::debug!("Adding post-context from diff [{current_idx}], lines [..{count}]");
            add_lines_to_diff(&mut result, text, ChangeType::Unchanged, Some((0, count)));

            last_processed_diff_idx = current_idx as i32;
            post_context_lines_from_prev_chunk = count;
        }
    }

    log::debug!(
        "contextual_diff returning result with {} lines",
        result.lines.len()
    );
    Ok(result)
}