liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

use std::path::Path;

use crate::constants::{MERGE_HEAD_FILE, ORIG_HEAD_FILE};
use crate::core::db;
use crate::core::merge;
use crate::error::OxenError;
use crate::model::{Commit, LocalRepository, merge_conflict::NodeMergeConflict};
use crate::util;
use crate::util::fs::AtomicFile;

use rocksdb::DB;

/// Creates the DB object then calls write_conflicts_to_disk
pub fn write_conflicts_to_db(
    repo: &LocalRepository,
    merge_commit: &Commit,
    base_commit: &Commit,
    conflicts: &[NodeMergeConflict],
) -> Result<(), OxenError> {
    let db_path = merge::db_path(repo);
    let opts = db::key_val::opts::default();
    let db = DB::open(&opts, dunce::simplified(&db_path))?;

    write_conflicts_to_disk(repo, &db, merge_commit, base_commit, conflicts)
}

/// Writes merge conflicts to database, and write merge commit and head commit to disk
pub fn write_conflicts_to_disk(
    repo: &LocalRepository,
    db: &DB,
    merge_commit: &Commit,
    base_commit: &Commit,
    conflicts: &[NodeMergeConflict],
) -> Result<(), OxenError> {
    // Write two files which are the merge commit and head commit so that we can make these parents later
    let hidden_dir = util::fs::oxen_hidden_dir(&repo.path);
    let merge_head_path = hidden_dir.join(MERGE_HEAD_FILE);
    let orig_head_path = hidden_dir.join(ORIG_HEAD_FILE);
    AtomicFile::new(&merge_head_path).write(merge_commit.id.as_bytes())?;
    AtomicFile::new(&orig_head_path).write(base_commit.id.as_bytes())?;

    for conflict in conflicts.iter() {
        let (_, base_path) = &conflict.base_entry;
        log::debug!("writing conflict to db: {base_path:?}");
        let key = base_path.to_str().unwrap();
        let key_bytes = key.as_bytes();
        let val_json = serde_json::to_string(&conflict)?;

        db.put(key_bytes, val_json.as_bytes())?;
    }

    Ok(())
}

pub fn mark_conflict_as_resolved_in_db(
    repo: &LocalRepository,
    path: impl AsRef<Path>,
) -> Result<(), OxenError> {
    let db_path = merge::db_path(repo);
    let opts = db::key_val::opts::default();
    let db = DB::open(&opts, dunce::simplified(&db_path))?;

    log::debug!(
        "mark_conflict_as_resolved_in_db path: {:?} db: {:?}",
        path.as_ref(),
        db_path
    );

    let key = path.as_ref().to_str().unwrap();
    let key_bytes = key.as_bytes();
    db.delete(key_bytes)?;

    Ok(())
}