1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
/// Parse an overlap threshold, rejecting values outside the documented
/// 0.0-1.0 range.
///
/// A threshold above 1.0 makes the per-sample overlap test unsatisfiable and
/// silently turns the AC-016 contamination gate into an unconditional pass;
/// below 0.0 it flags everything. Neither is a meaningful ratio.
fn parse_unit_interval(raw: &str) -> Result<f64, String> {
let value: f64 = raw
.parse()
.map_err(|_| format!("'{raw}' is not a number"))?;
if !value.is_finite() || !(0.0..=1.0).contains(&value) {
return Err(format!(
"'{raw}' is outside the valid range 0.0-1.0 (an overlap ratio)"
));
}
Ok(value)
}
/// Parse an n-gram size, rejecting 0.
///
/// A zero-width window is not a window: `--ngram 0` reached
/// `slice::windows(0)` inside the decontamination scan and aborted the
/// process with "window size must be non-zero" (exit 101).
fn parse_ngram_size(raw: &str) -> Result<usize, String> {
let value: usize = raw
.parse()
.map_err(|_| format!("'{raw}' is not a non-negative integer"))?;
if value == 0 {
return Err("n-gram size must be >= 1; a zero-width window compares nothing".to_string());
}
Ok(value)
}
/// Data quality pipeline subcommands (powered by alimentar).
///
/// Thin CLI wrappers around alimentar's data utilities.
#[derive(Subcommand, Debug)]
pub enum DataCommands {
/// Every alimentar data command: convert, info, head, schema, mix, fim,
/// filter-text, view, import, hub, registry, drift, quality, fed, doctest,
/// extract, merge.
///
/// APR-MONO consolidated alimentar in-tree, but its capability stayed
/// reachable only through the standalone `alimentar` binary -- `apr data`
/// shipped 5 commands against alimentar's 20, so 18 had no route through
/// `apr` at all. This dispatches the SAME `alimentar::cli::dispatch`, so
/// there is one implementation behind two names rather than a second clap
/// tree that can drift from the first.
#[command(subcommand, name = "x")]
Alimentar(alimentar::cli::Commands),
/// Audit a JSONL classification dataset for quality issues
Audit {
/// Path to JSONL data file
#[arg(value_name = "FILE")]
file: PathBuf,
/// Number of output classes (for label range validation)
#[arg(long, default_value = "5")]
num_classes: usize,
/// Input text column name
#[arg(long, default_value = "input")]
input_column: String,
/// Label column name
#[arg(long, default_value = "label")]
label_column: String,
/// Preamble prefix to detect (e.g., "#!/")
#[arg(long, default_value = "#!/")]
preamble_prefix: Option<String>,
},
/// Stratified train/val/test split preserving class proportions
Split {
/// Path to JSONL data file
#[arg(value_name = "FILE")]
file: PathBuf,
/// Training set fraction
#[arg(long, default_value = "0.8")]
train: f64,
/// Validation set fraction
#[arg(long, default_value = "0.1")]
val: f64,
/// Test set fraction
#[arg(long, default_value = "0.1")]
test: f64,
/// Label column name for stratification
#[arg(long, default_value = "label")]
label_column: String,
/// Random seed for deterministic split
#[arg(long, default_value = "42")]
seed: u64,
/// Output directory for split files
#[arg(short, long)]
output: PathBuf,
},
/// Check training data for benchmark contamination via n-gram overlap
Decontaminate {
/// Path to training JSONL data file
#[arg(value_name = "FILE")]
file: PathBuf,
/// Reference benchmark JSONL files to check against
#[arg(long, required = true, num_args = 1..)]
reference: Vec<PathBuf>,
/// N-gram size for overlap detection (must be >= 1)
#[arg(long, default_value = "10", value_parser = parse_ngram_size)]
ngram: usize,
/// Overlap threshold (0.0-1.0) above which a sample is flagged
#[arg(long, default_value = "0.5", value_parser = parse_unit_interval)]
threshold: f64,
/// Output as JSON
#[arg(long)]
json: bool,
},
/// Remove exact duplicate rows from a JSONL dataset
Dedup {
/// Path to JSONL data file
#[arg(value_name = "FILE")]
file: PathBuf,
/// Output file path for the deduplicated dataset
#[arg(short, long)]
output: PathBuf,
/// Output as JSON
#[arg(long)]
json: bool,
},
/// Resample dataset to address class imbalance
Balance {
/// Path to JSONL data file
#[arg(value_name = "FILE")]
file: PathBuf,
/// Rebalancing strategy: oversample, undersample, sqrt-inverse
#[arg(long, default_value = "oversample")]
strategy: String,
/// Label column name
#[arg(long, default_value = "label")]
label_column: String,
/// Number of classes (for sqrt-inverse weight computation)
#[arg(long)]
num_classes: Option<usize>,
/// Random seed
#[arg(long, default_value = "42")]
seed: u64,
/// Output file path (required for oversample/undersample)
#[arg(short, long)]
output: Option<PathBuf>,
},
}