1use std::collections::BTreeMap;
2
3use polars::prelude::{DataFrame, NamedFrom, Series, SortMultipleOptions};
4
5use crate::data::validate_factor_frame;
6use crate::error::{FerricAlphaError, Result};
7
8#[derive(Debug, Clone, PartialEq)]
9pub struct QuantizeOptions {
10 quantiles: Option<u32>,
11 quantile_edges: Option<Vec<f64>>,
12 bin_count: Option<u32>,
13 bins: Option<Vec<f64>>,
14 by_group: bool,
15 zero_aware: bool,
16}
17
18impl QuantizeOptions {
19 pub fn quantiles(quantiles: u32) -> Self {
20 Self {
21 quantiles: Some(quantiles),
22 quantile_edges: None,
23 bin_count: None,
24 bins: None,
25 by_group: false,
26 zero_aware: false,
27 }
28 }
29
30 pub fn quantile_edges(edges: Vec<f64>) -> Self {
31 Self {
32 quantiles: None,
33 quantile_edges: Some(edges),
34 bin_count: None,
35 bins: None,
36 by_group: false,
37 zero_aware: false,
38 }
39 }
40
41 pub fn bin_count(bins: u32) -> Self {
42 Self {
43 quantiles: None,
44 quantile_edges: None,
45 bin_count: Some(bins),
46 bins: None,
47 by_group: false,
48 zero_aware: false,
49 }
50 }
51
52 pub fn bins(bins: Vec<f64>) -> Self {
53 Self {
54 quantiles: None,
55 quantile_edges: None,
56 bin_count: None,
57 bins: Some(bins),
58 by_group: false,
59 zero_aware: false,
60 }
61 }
62
63 pub fn with_by_group(mut self, by_group: bool) -> Self {
64 self.by_group = by_group;
65 self
66 }
67
68 pub fn with_zero_aware(mut self, zero_aware: bool) -> Self {
69 self.zero_aware = zero_aware;
70 self
71 }
72
73 fn mode(&self) -> Result<QuantizeMode<'_>> {
74 let active_modes = u8::from(self.quantiles.is_some())
75 + u8::from(self.quantile_edges.is_some())
76 + u8::from(self.bin_count.is_some())
77 + u8::from(self.bins.is_some());
78 if active_modes > 1 {
79 return Err(FerricAlphaError::ConflictingQuantileMode);
80 }
81 if let Some(quantiles) = self.quantiles {
82 if quantiles < 2 {
83 return Err(FerricAlphaError::InvalidQuantiles {
84 reason: "quantiles must be at least 2",
85 });
86 }
87 if self.zero_aware && quantiles % 2 != 0 {
88 return Err(FerricAlphaError::InvalidQuantiles {
89 reason: "zero-aware quantiles must be even",
90 });
91 }
92 return Ok(QuantizeMode::Quantiles(quantiles));
93 }
94
95 if let Some(edges) = self.quantile_edges.as_deref() {
96 if edges.len() < 2 {
97 return Err(FerricAlphaError::InvalidQuantiles {
98 reason: "quantile edges must contain at least two boundaries",
99 });
100 }
101 for edge in edges {
102 if !edge.is_finite() || *edge < 0.0 || *edge > 1.0 {
103 return Err(FerricAlphaError::InvalidQuantiles {
104 reason: "quantile edges must be finite values in [0, 1]",
105 });
106 }
107 }
108 for pair in edges.windows(2) {
109 if pair[0] >= pair[1] {
110 return Err(FerricAlphaError::InvalidQuantiles {
111 reason: "quantile edges must be strictly increasing",
112 });
113 }
114 }
115 return Ok(QuantizeMode::QuantileEdges(edges));
116 }
117
118 if let Some(bin_count) = self.bin_count {
119 if bin_count < 1 {
120 return Err(FerricAlphaError::InvalidQuantiles {
121 reason: "bins must be at least 1",
122 });
123 }
124 if self.zero_aware && bin_count % 2 != 0 {
125 return Err(FerricAlphaError::InvalidQuantiles {
126 reason: "zero-aware bins must be even",
127 });
128 }
129 return Ok(QuantizeMode::BinCount(bin_count));
130 }
131
132 let bins = self
133 .bins
134 .as_deref()
135 .ok_or(FerricAlphaError::InvalidQuantiles {
136 reason: "quantiles or bins are required",
137 })?;
138 if bins.len() < 2 {
139 return Err(FerricAlphaError::InvalidQuantiles {
140 reason: "bins must contain at least two boundaries",
141 });
142 }
143 if self.zero_aware {
144 return Err(FerricAlphaError::InvalidQuantiles {
145 reason: "zero-aware explicit bins are not supported in this phase",
146 });
147 }
148 for pair in bins.windows(2) {
149 if !pair[0].is_finite() || !pair[1].is_finite() || pair[0] >= pair[1] {
150 return Err(FerricAlphaError::InvalidQuantiles {
151 reason: "bin boundaries must be finite and strictly increasing",
152 });
153 }
154 }
155 Ok(QuantizeMode::Bins(bins))
156 }
157}
158
159#[derive(Debug, Clone, Copy)]
160enum QuantizeMode<'a> {
161 Quantiles(u32),
162 QuantileEdges(&'a [f64]),
163 BinCount(u32),
164 Bins(&'a [f64]),
165}
166
167pub fn quantize_factor(factor: &DataFrame, options: QuantizeOptions) -> Result<DataFrame> {
168 let mode = options.mode()?;
169 let factor = validate_factor_frame(factor)?;
170 if options.by_group && factor.column("group").is_err() {
171 return Err(FerricAlphaError::MissingColumn("group"));
172 }
173
174 let dates = factor.column("date")?.as_materialized_series().datetime()?;
175 let assets = factor.column("asset")?.as_materialized_series().str()?;
176 let factors = factor.column("factor")?.as_materialized_series().f64()?;
177 let groups = if options.by_group {
178 Some(factor.column("group")?.as_materialized_series().str()?)
179 } else {
180 None
181 };
182
183 let mut rows = Vec::with_capacity(factor.height());
184 let mut partitions: BTreeMap<(i64, Option<String>), Vec<usize>> = BTreeMap::new();
185 for index in 0..factor.height() {
186 let date = dates
187 .physical()
188 .get(index)
189 .expect("validated factor date should not be null");
190 let asset = assets
191 .get(index)
192 .expect("validated factor asset should not be null")
193 .to_owned();
194 let group = groups
195 .as_ref()
196 .and_then(|groups| groups.get(index).map(ToOwned::to_owned));
197 let factor_value = factors.get(index).filter(|value| value.is_finite());
198 rows.push(QuantileRow {
199 asset,
200 group: group.clone(),
201 factor: factor_value,
202 });
203 if factor_value.is_some() {
204 partitions.entry((date, group)).or_default().push(index);
205 }
206 }
207
208 let mut assignments = vec![None; rows.len()];
209 for indexes in partitions.values() {
210 match mode {
211 QuantizeMode::Quantiles(quantiles) if options.zero_aware => {
212 assign_zero_aware(indexes, &rows, quantiles, &mut assignments);
213 }
214 QuantizeMode::Quantiles(quantiles) => {
215 assign_quantiles(indexes, &rows, quantiles, 1, &mut assignments);
216 }
217 QuantizeMode::QuantileEdges(edges) => {
218 if options.zero_aware {
219 assign_zero_aware_edges(indexes, &rows, edges, &mut assignments);
220 } else {
221 assign_quantile_edges(indexes, &rows, edges, 1, &mut assignments);
222 }
223 }
224 QuantizeMode::BinCount(bin_count) if options.zero_aware => {
225 assign_zero_aware_bin_count(indexes, &rows, bin_count, &mut assignments);
226 }
227 QuantizeMode::BinCount(bin_count) => {
228 assign_bin_count(indexes, &rows, bin_count, 1, &mut assignments);
229 }
230 QuantizeMode::Bins(bins) => {
231 assign_bins(indexes, &rows, bins, false, &mut assignments);
232 }
233 }
234 }
235
236 let mut columns = factor.columns().to_vec();
237 columns.push(Series::new("factor_quantile".into(), assignments).into());
238 Ok(DataFrame::new(factor.height(), columns)?
239 .sort(["date", "asset"], SortMultipleOptions::default())?)
240}
241
242fn assign_bins(
243 indexes: &[usize],
244 rows: &[QuantileRow],
245 bins: &[f64],
246 include_lowest: bool,
247 assignments: &mut [Option<u32>],
248) {
249 for index in indexes {
250 let value = rows[*index].factor.expect("valid row should have factor");
251 let bucket = bins
252 .windows(2)
253 .enumerate()
254 .position(|(bucket, pair)| {
255 (value > pair[0] || (include_lowest && bucket == 0 && value == pair[0]))
256 && value <= pair[1]
257 })
258 .map(|bucket| bucket as u32 + 1);
259 assignments[*index] = bucket;
260 }
261}
262
263fn assign_bin_count(
264 indexes: &[usize],
265 rows: &[QuantileRow],
266 bin_count: u32,
267 offset: u32,
268 assignments: &mut [Option<u32>],
269) {
270 if indexes.is_empty() {
271 return;
272 }
273 let values = indexes
274 .iter()
275 .map(|index| rows[*index].factor.expect("valid row should have factor"))
276 .collect::<Vec<_>>();
277 let min = values.iter().copied().reduce(f64::min).unwrap();
278 let max = values.iter().copied().reduce(f64::max).unwrap();
279 if bin_count == 1 || min == max {
280 for index in indexes {
281 assignments[*index] = Some(offset);
282 }
283 return;
284 }
285 let width = (max - min) / bin_count as f64;
286 for index in indexes {
287 let value = rows[*index].factor.expect("valid row should have factor");
288 let bucket = if value == min {
289 1
290 } else {
291 ((value - min) / width).ceil().clamp(1.0, bin_count as f64) as u32
292 };
293 assignments[*index] = Some(offset + bucket - 1);
294 }
295}
296
297#[derive(Debug)]
298struct QuantileRow {
299 asset: String,
300 group: Option<String>,
301 factor: Option<f64>,
302}
303
304fn assign_zero_aware(
305 indexes: &[usize],
306 rows: &[QuantileRow],
307 quantiles: u32,
308 assignments: &mut [Option<u32>],
309) {
310 let midpoint = quantiles / 2;
311 let negative = indexes
312 .iter()
313 .copied()
314 .filter(|index| rows[*index].factor.expect("valid row should have factor") < 0.0)
315 .collect::<Vec<_>>();
316 let positive = indexes
317 .iter()
318 .copied()
319 .filter(|index| rows[*index].factor.expect("valid row should have factor") >= 0.0)
320 .collect::<Vec<_>>();
321 assign_quantiles(&negative, rows, midpoint, 1, assignments);
322 assign_quantiles(&positive, rows, midpoint, midpoint + 1, assignments);
323}
324
325fn assign_zero_aware_edges(
326 indexes: &[usize],
327 rows: &[QuantileRow],
328 edges: &[f64],
329 assignments: &mut [Option<u32>],
330) {
331 let bucket_count = edges.len() as u32 - 1;
332 let midpoint = bucket_count / 2;
333 let negative = indexes
334 .iter()
335 .copied()
336 .filter(|index| rows[*index].factor.expect("valid row should have factor") < 0.0)
337 .collect::<Vec<_>>();
338 let positive = indexes
339 .iter()
340 .copied()
341 .filter(|index| rows[*index].factor.expect("valid row should have factor") >= 0.0)
342 .collect::<Vec<_>>();
343 assign_quantile_edges(&negative, rows, edges, 1, assignments);
344 assign_quantile_edges(&positive, rows, edges, midpoint + 1, assignments);
345}
346
347fn assign_zero_aware_bin_count(
348 indexes: &[usize],
349 rows: &[QuantileRow],
350 bin_count: u32,
351 assignments: &mut [Option<u32>],
352) {
353 let midpoint = bin_count / 2;
354 let negative = indexes
355 .iter()
356 .copied()
357 .filter(|index| rows[*index].factor.expect("valid row should have factor") < 0.0)
358 .collect::<Vec<_>>();
359 let positive = indexes
360 .iter()
361 .copied()
362 .filter(|index| rows[*index].factor.expect("valid row should have factor") >= 0.0)
363 .collect::<Vec<_>>();
364 assign_bin_count(&negative, rows, midpoint, 1, assignments);
365 assign_bin_count(&positive, rows, midpoint, midpoint + 1, assignments);
366}
367
368fn assign_quantile_edges(
369 indexes: &[usize],
370 rows: &[QuantileRow],
371 edges: &[f64],
372 offset: u32,
373 assignments: &mut [Option<u32>],
374) {
375 if indexes.is_empty() {
376 return;
377 }
378
379 let mut values = indexes
380 .iter()
381 .map(|index| rows[*index].factor.expect("valid row should have factor"))
382 .collect::<Vec<_>>();
383 values.sort_by(f64::total_cmp);
384 let cut_points = edges
385 .iter()
386 .map(|edge| percentile(&values, *edge))
387 .collect::<Vec<_>>();
388 assign_bins(indexes, rows, &cut_points, true, assignments);
389 for index in indexes {
390 if let Some(value) = &mut assignments[*index]
391 && *value > 0
392 {
393 *value += offset - 1;
394 }
395 }
396}
397
398fn assign_quantiles(
399 indexes: &[usize],
400 rows: &[QuantileRow],
401 quantiles: u32,
402 offset: u32,
403 assignments: &mut [Option<u32>],
404) {
405 if indexes.is_empty() {
406 return;
407 }
408 if indexes.len() < 2 {
409 return;
410 }
411
412 let mut sorted = indexes.to_vec();
413 sorted.sort_by(|left, right| {
414 let left_row = &rows[*left];
415 let right_row = &rows[*right];
416 left_row
417 .factor
418 .partial_cmp(&right_row.factor)
419 .expect("finite factors should be comparable")
420 .then_with(|| left_row.group.cmp(&right_row.group))
421 .then_with(|| left_row.asset.cmp(&right_row.asset))
422 });
423
424 for (rank, index) in sorted.iter().enumerate() {
425 let bucket = if sorted.len() >= quantiles as usize {
426 ((rank as u32 * quantiles) / sorted.len() as u32).min(quantiles - 1)
427 } else {
428 let scaled_rank = rank as f64 * (quantiles - 1) as f64 / (sorted.len() - 1) as f64;
429 scaled_rank.round() as u32
430 };
431 assignments[*index] = Some(offset + bucket);
432 }
433}
434
435fn percentile(sorted: &[f64], probability: f64) -> f64 {
436 if sorted.len() == 1 {
437 return sorted[0];
438 }
439 let position = probability * (sorted.len() - 1) as f64;
440 let lower = position.floor() as usize;
441 let upper = position.ceil() as usize;
442 if lower == upper {
443 sorted[lower]
444 } else {
445 let weight = position - lower as f64;
446 sorted[lower] * (1.0 - weight) + sorted[upper] * weight
447 }
448}