liboxen 0.9.9-alpha

Oxen is a fast, unstructured data version control, to help version datasets, written in Rust.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

//! Computes metadata we can extract from the entry files

use crate::constants::{CACHE_DIR, DIRS_DIR, HISTORY_DIR};
use crate::core::df::tabular;
use crate::core::index::{commit_metadata_db, CommitEntryReader};
use crate::error::OxenError;
use crate::model::{Commit, LocalRepository};
use crate::util;

use std::path::{Path, PathBuf};

pub fn dir_column_path(
    repo: &LocalRepository,
    commit: &Commit,
    dir: &Path,
    column: &str,
) -> PathBuf {
    util::fs::oxen_hidden_dir(&repo.path)
        .join(HISTORY_DIR)
        .join(&commit.id)
        .join(CACHE_DIR)
        .join(DIRS_DIR)
        .join(dir)
        .join(format!("{}.parquet", column))
}

pub fn compute(repo: &LocalRepository, commit: &Commit) -> Result<(), OxenError> {
    log::debug!("Running content_metadata");

    log::debug!("computing metadata {} -> {}", commit.id, commit.message);

    // Compute the metadata stats
    commit_metadata_db::index_commit(repo, commit)?;

    // Then for each directory, aggregate up the data_type and mime_type, and save as a dataframe file
    // that we can serve up.
    let reader = CommitEntryReader::new(repo, commit)?;
    let dirs = reader.list_dirs()?;
    log::debug!("Computing size of {} dirs", dirs.len());
    let columns = ["data_type", "mime_type"];
    for dir in dirs {
        for column in columns.iter() {
            log::debug!("Aggregating {column} for commit {commit:?}");
            let mut df = commit_metadata_db::aggregate_col(repo, commit, &dir, column)?;
            let path = dir_column_path(repo, commit, &dir, column);
            if let Some(parent) = path.parent() {
                std::fs::create_dir_all(parent)?;
            }
            log::debug!("Writing cached df {} to {}", column, path.display());
            // log::debug!("df {}", df);
            tabular::write_df(&mut df, &path)?;
        }
    }

    Ok(())
}