liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
use async_compression::futures::bufread::GzipDecoder;
use async_tar::Archive;
use std::path::Path;
use tokio_util::compat::TokioAsyncReadCompatExt;

use crate::constants::DEFAULT_BRANCH_NAME;
use crate::core::v_latest::branches::OnConflict;
use crate::repositories;
use crate::util;
use crate::{error::OxenError, model::LocalRepository};
pub async fn load(
    src_path: &Path,
    dest_path: &Path,
    no_working_dir: bool,
) -> Result<(), OxenError> {
    let done_msg: String = format!("✅ Loaded {src_path:?} to an oxen repo at {dest_path:?}");

    let dest_path = if dest_path.exists() {
        if dest_path.is_file() {
            return Err(OxenError::basic_str(
                "Destination path is a file, must be a directory",
            ));
        }
        dest_path.to_path_buf()
    } else {
        util::fs::create_dir_all(dest_path)?;
        dest_path.to_path_buf()
    };

    let file = tokio::fs::File::open(src_path).await?;
    let reader = futures::io::BufReader::new(file.compat());
    let decoder = GzipDecoder::new(reader);
    let archive = Archive::new(decoder);
    println!("🐂 Decompressing oxen repo into {dest_path:?}");
    util::fs::unpack_async_tar_archive(archive, &dest_path).await?;

    // Server repos - done unpacking
    if no_working_dir {
        println!("{done_msg}");
        return Ok(());
    }

    // Client repos - need to hydrate working dir from versions files
    let repo = LocalRepository::from_dir(&dest_path)?;

    println!("🐂 Unpacking files to working directory {dest_path:?}");
    let branch = repositories::branches::get_by_name(&repo, DEFAULT_BRANCH_NAME)?;
    let commit = repositories::commits::get_by_id(&repo, &branch.commit_id)?
        .ok_or_else(|| OxenError::commit_id_does_not_exist(&branch.commit_id))?;
    repositories::branches::set_working_repo_to_commit(&repo, &commit, &None, OnConflict::Abort)
        .await?;

    println!("{done_msg}");
    Ok(())
}

#[cfg(test)]
mod tests {
    use std::path::Path;

    use crate::error::OxenError;
    use crate::model::LocalRepository;
    use crate::repositories;
    use crate::test;
    use crate::util;

    #[tokio::test]
    async fn test_command_save_repo() -> Result<(), OxenError> {
        test::run_empty_local_repo_test_async(|repo| async move {
            // Write one file
            let hello_file = repo.path.join("hello.txt");
            util::fs::write_to_path(&hello_file, "Hello World")?;
            // Add-commit
            repositories::add(&repo, &hello_file).await?;
            repositories::commit(&repo, "Adding hello file")?;

            // Save to a path
            let save_path = repo.path.join(Path::new("backup.tar.gz"));
            repositories::save(&repo, &save_path)?;

            assert!(save_path.exists());

            // Cleanup tarball
            util::fs::remove_file(save_path)?;

            Ok(())
        })
        .await
    }

    #[tokio::test]
    async fn test_command_save_load_repo_with_working_dir() -> Result<(), OxenError> {
        test::run_empty_local_repo_test_async(|repo| async move {
            test::run_empty_dir_test_async(|dir| async move {
                // Write one file
                let hello_file = repo.path.join("hello.txt");
                util::fs::write_to_path(&hello_file, "Hello World")?;
                // Add-commit
                repositories::add(&repo, &hello_file).await?;
                repositories::commit(&repo, "Adding hello file")?;

                // Save to a path
                let save_path = dir.join(Path::new("backup.tar.gz"));
                repositories::save(&repo, &save_path)?;

                // Load from a path and hydrate
                let loaded_repo_path = dir.join(Path::new("loaded_repo"));
                repositories::load(&save_path, &loaded_repo_path, false).await?;

                let hydrated_repo = LocalRepository::from_dir(&loaded_repo_path)?;
                assert!(hydrated_repo.path.join("hello.txt").exists());

                // Cleanup tarball
                util::fs::remove_file(save_path)?;

                Ok(())
            })
            .await
        })
        .await
    }

    #[tokio::test]
    async fn test_command_save_load_repo_no_working_dir() -> Result<(), OxenError> {
        test::run_empty_local_repo_test_async(|repo| async move {
            test::run_empty_dir_test_async(|dir| async move {
                // Write one file
                let hello_file = repo.path.join("hello.txt");
                util::fs::write_to_path(&hello_file, "Hello World")?;
                // Add-commit
                repositories::add(&repo, &hello_file).await?;
                repositories::commit(&repo, "Adding hello file")?;

                // Save to a path
                let save_path = dir.join(Path::new("backup.tar.gz"));
                repositories::save(&repo, &save_path)?;

                // Load from a path and hydrate
                let loaded_repo_path = dir.join(Path::new("loaded_repo"));
                repositories::load(&save_path, &loaded_repo_path, true).await?;

                let hydrated_repo = LocalRepository::from_dir(&loaded_repo_path)?;

                assert!(!hydrated_repo.path.join("hello.txt").exists());

                // Should have `hello.txt` in removed files bc it's in commits db but not working dir
                let status = repositories::status(&hydrated_repo).await?;

                assert_eq!(status.removed_files.len(), 1);

                // Cleanup tarball
                util::fs::remove_file(save_path)?;

                Ok(())
            })
            .await
        })
        .await
    }

    #[tokio::test]
    async fn test_command_save_load_moved_and_removed() -> Result<(), OxenError> {
        test::run_empty_local_repo_test_async(|repo| async move {
            test::run_empty_dir_test_async(|dir| async move {
                // Write one file
                let hello_file = repo.path.join("hello.txt");
                let goodbye_file = repo.path.join("goodbye.txt");
                util::fs::write_to_path(&hello_file, "Hello World")?;
                util::fs::write_to_path(&goodbye_file, "Goodbye World")?;
                // Add-commit
                repositories::add(&repo, &hello_file).await?;
                repositories::add(&repo, &goodbye_file).await?;
                repositories::commit(&repo, "Adding hello file")?;

                // Move hello into a folder
                let hello_dir = repo.path.join("hello_dir");
                std::fs::create_dir(&hello_dir)?;
                let moved_hello = hello_dir.join("hello.txt");
                util::fs::rename(&hello_file, &moved_hello)?;

                // Remove goodbye
                std::fs::remove_file(&goodbye_file)?;

                // Add a third file
                let third_file = repo.path.join("third.txt");
                util::fs::write_to_path(&third_file, "Third File")?;

                // Add-commit
                repositories::add(&repo, &moved_hello).await?;
                repositories::add(&repo, &hello_file).await?;
                repositories::add(&repo, &goodbye_file).await?;
                repositories::add(&repo, &third_file).await?;
                repositories::commit(&repo, "Moving hello file")?;

                // Save to a path
                let save_path = dir.join(Path::new("backup.tar.gz"));
                repositories::save(&repo, &save_path)?;

                // Load from a path and hydrate
                let loaded_repo_path = dir.join(Path::new("loaded_repo"));
                repositories::load(&save_path, &loaded_repo_path, false).await?;

                let hydrated_repo = LocalRepository::from_dir(&loaded_repo_path)?;

                // List files in repo
                let files = util::fs::rlist_files_in_dir(&hydrated_repo.path);
                println!("Files in hydrated repo: {files:?}");

                assert!(hydrated_repo.path.join("third.txt").exists());
                assert!(hydrated_repo.path.join("hello_dir/hello.txt").exists());
                assert!(!hydrated_repo.path.join("hello.txt").exists());
                assert!(!hydrated_repo.path.join("goodbye.txt").exists());

                // Cleanup tarball
                util::fs::remove_file(save_path)?;

                Ok(())
            })
            .await
        })
        .await
    }
}