apr_cli/
data_commands.rs

1
2/// Parse an overlap threshold, rejecting values outside the documented
3/// 0.0-1.0 range.
4///
5/// A threshold above 1.0 makes the per-sample overlap test unsatisfiable and
6/// silently turns the AC-016 contamination gate into an unconditional pass;
7/// below 0.0 it flags everything. Neither is a meaningful ratio.
8fn parse_unit_interval(raw: &str) -> Result<f64, String> {
9    let value: f64 = raw
10        .parse()
11        .map_err(|_| format!("'{raw}' is not a number"))?;
12    if !value.is_finite() || !(0.0..=1.0).contains(&value) {
13        return Err(format!(
14            "'{raw}' is outside the valid range 0.0-1.0 (an overlap ratio)"
15        ));
16    }
17    Ok(value)
18}
19
20/// Parse an n-gram size, rejecting 0.
21///
22/// A zero-width window is not a window: `--ngram 0` reached
23/// `slice::windows(0)` inside the decontamination scan and aborted the
24/// process with "window size must be non-zero" (exit 101).
25fn parse_ngram_size(raw: &str) -> Result<usize, String> {
26    let value: usize = raw
27        .parse()
28        .map_err(|_| format!("'{raw}' is not a non-negative integer"))?;
29    if value == 0 {
30        return Err("n-gram size must be >= 1; a zero-width window compares nothing".to_string());
31    }
32    Ok(value)
33}
34
35/// Data quality pipeline subcommands (powered by alimentar).
36///
37/// Thin CLI wrappers around alimentar's data utilities.
38#[derive(Subcommand, Debug)]
39pub enum DataCommands {
40    /// Every alimentar data command: convert, info, head, schema, mix, fim,
41    /// filter-text, view, import, hub, registry, drift, quality, fed, doctest,
42    /// extract, merge.
43    ///
44    /// APR-MONO consolidated alimentar in-tree, but its capability stayed
45    /// reachable only through the standalone `alimentar` binary -- `apr data`
46    /// shipped 5 commands against alimentar's 20, so 18 had no route through
47    /// `apr` at all. This dispatches the SAME `alimentar::cli::dispatch`, so
48    /// there is one implementation behind two names rather than a second clap
49    /// tree that can drift from the first.
50    #[command(subcommand, name = "x")]
51    Alimentar(alimentar::cli::Commands),
52
53    /// Audit a JSONL classification dataset for quality issues
54    Audit {
55        /// Path to JSONL data file
56        #[arg(value_name = "FILE")]
57        file: PathBuf,
58        /// Number of output classes (for label range validation)
59        #[arg(long, default_value = "5")]
60        num_classes: usize,
61        /// Input text column name
62        #[arg(long, default_value = "input")]
63        input_column: String,
64        /// Label column name
65        #[arg(long, default_value = "label")]
66        label_column: String,
67        /// Preamble prefix to detect (e.g., "#!/")
68        #[arg(long, default_value = "#!/")]
69        preamble_prefix: Option<String>,
70    },
71    /// Stratified train/val/test split preserving class proportions
72    Split {
73        /// Path to JSONL data file
74        #[arg(value_name = "FILE")]
75        file: PathBuf,
76        /// Training set fraction
77        #[arg(long, default_value = "0.8")]
78        train: f64,
79        /// Validation set fraction
80        #[arg(long, default_value = "0.1")]
81        val: f64,
82        /// Test set fraction
83        #[arg(long, default_value = "0.1")]
84        test: f64,
85        /// Label column name for stratification
86        #[arg(long, default_value = "label")]
87        label_column: String,
88        /// Random seed for deterministic split
89        #[arg(long, default_value = "42")]
90        seed: u64,
91        /// Output directory for split files
92        #[arg(short, long)]
93        output: PathBuf,
94    },
95    /// Check training data for benchmark contamination via n-gram overlap
96    Decontaminate {
97        /// Path to training JSONL data file
98        #[arg(value_name = "FILE")]
99        file: PathBuf,
100        /// Reference benchmark JSONL files to check against
101        #[arg(long, required = true, num_args = 1..)]
102        reference: Vec<PathBuf>,
103        /// N-gram size for overlap detection (must be >= 1)
104        #[arg(long, default_value = "10", value_parser = parse_ngram_size)]
105        ngram: usize,
106        /// Overlap threshold (0.0-1.0) above which a sample is flagged
107        #[arg(long, default_value = "0.5", value_parser = parse_unit_interval)]
108        threshold: f64,
109        /// Output as JSON
110        #[arg(long)]
111        json: bool,
112    },
113    /// Remove exact duplicate rows from a JSONL dataset
114    Dedup {
115        /// Path to JSONL data file
116        #[arg(value_name = "FILE")]
117        file: PathBuf,
118        /// Output file path for the deduplicated dataset
119        #[arg(short, long)]
120        output: PathBuf,
121        /// Output as JSON
122        #[arg(long)]
123        json: bool,
124    },
125    /// Resample dataset to address class imbalance
126    Balance {
127        /// Path to JSONL data file
128        #[arg(value_name = "FILE")]
129        file: PathBuf,
130        /// Rebalancing strategy: oversample, undersample, sqrt-inverse
131        #[arg(long, default_value = "oversample")]
132        strategy: String,
133        /// Label column name
134        #[arg(long, default_value = "label")]
135        label_column: String,
136        /// Number of classes (for sqrt-inverse weight computation)
137        #[arg(long)]
138        num_classes: Option<usize>,
139        /// Random seed
140        #[arg(long, default_value = "42")]
141        seed: u64,
142        /// Output file path (required for oversample/undersample)
143        #[arg(short, long)]
144        output: Option<PathBuf>,
145    },
146}
apr_cli/data_commands.rs

apr_cli/
data_commands.rs
