palace 0.1.0

A tool for mounting datasets into memory for fast loading in deep learning tasks.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

use crate::mount::{mount_dataset, MountOptions};
use crate::unmount::{unmount_dataset, UnmountOptions};
use anyhow::Result;
use clap::{Parser, Subcommand};
use std::path::PathBuf;

#[derive(Parser)]
#[clap(author, version, about, long_about = None)]
struct Cli {
    #[clap(subcommand)]
    command: Commands,
}

#[derive(Subcommand)]
enum Commands {
    /// Mount a dataset to memory
    Mount {
        /// Source directory of the dataset
        #[clap(short, long)]
        source: PathBuf,

        /// Destination directory for the mounted dataset
        #[clap(short, long)]
        destination: PathBuf,

        /// Number of parallel operations
        #[clap(short, long, default_value = "4")]
        parallel: usize,

        /// Use memory mapping instead of file copy
        #[clap(long)]
        mmap: bool,

        /// Compress data while mounting
        #[clap(short, long)]
        compress: bool,

        /// Memory threshold percentage (0-100)
        #[clap(short, long, default_value = "90")]
        threshold: u8,
    },

    /// Unmount a dataset from memory
    Unmount {
        /// Target directory of the mounted dataset
        #[clap(short, long)]
        target: PathBuf,

        /// Force unmount without confirmation
        #[clap(short, long)]
        force: bool,
    },
}

pub fn run() -> Result<()> {
    let cli = Cli::parse();

    match &cli.command {
        Commands::Mount {
            source,
            destination,
            parallel,
            mmap,
            compress,
            threshold,
        } => {
            let options = MountOptions {
                source: source.clone(),
                destination: destination.clone(),
                parallel: *parallel,
                use_mmap: *mmap,
                compress: *compress,
                memory_threshold: *threshold,
            };
            mount_dataset(&options)?;
            println!("Dataset mounted successfully.");
        }
        Commands::Unmount { target, force } => {
            let options = UnmountOptions {
                target: target.clone(),
                force: *force,
            };
            unmount_dataset(&options)?;
            println!("Dataset unmounted successfully.");
        }
    }

    Ok(())
}