Skip to main content

ferric_alpha/
quantiles.rs

1use std::collections::BTreeMap;
2
3use polars::prelude::{DataFrame, NamedFrom, Series, SortMultipleOptions};
4
5use crate::data::validate_factor_frame;
6use crate::error::{FerricAlphaError, Result};
7
8#[derive(Debug, Clone, PartialEq)]
9pub struct QuantizeOptions {
10    quantiles: Option<u32>,
11    quantile_edges: Option<Vec<f64>>,
12    bin_count: Option<u32>,
13    bins: Option<Vec<f64>>,
14    by_group: bool,
15    zero_aware: bool,
16}
17
18impl QuantizeOptions {
19    pub fn quantiles(quantiles: u32) -> Self {
20        Self {
21            quantiles: Some(quantiles),
22            quantile_edges: None,
23            bin_count: None,
24            bins: None,
25            by_group: false,
26            zero_aware: false,
27        }
28    }
29
30    pub fn quantile_edges(edges: Vec<f64>) -> Self {
31        Self {
32            quantiles: None,
33            quantile_edges: Some(edges),
34            bin_count: None,
35            bins: None,
36            by_group: false,
37            zero_aware: false,
38        }
39    }
40
41    pub fn bin_count(bins: u32) -> Self {
42        Self {
43            quantiles: None,
44            quantile_edges: None,
45            bin_count: Some(bins),
46            bins: None,
47            by_group: false,
48            zero_aware: false,
49        }
50    }
51
52    pub fn bins(bins: Vec<f64>) -> Self {
53        Self {
54            quantiles: None,
55            quantile_edges: None,
56            bin_count: None,
57            bins: Some(bins),
58            by_group: false,
59            zero_aware: false,
60        }
61    }
62
63    pub fn with_by_group(mut self, by_group: bool) -> Self {
64        self.by_group = by_group;
65        self
66    }
67
68    pub fn with_zero_aware(mut self, zero_aware: bool) -> Self {
69        self.zero_aware = zero_aware;
70        self
71    }
72
73    fn mode(&self) -> Result<QuantizeMode<'_>> {
74        let active_modes = u8::from(self.quantiles.is_some())
75            + u8::from(self.quantile_edges.is_some())
76            + u8::from(self.bin_count.is_some())
77            + u8::from(self.bins.is_some());
78        if active_modes > 1 {
79            return Err(FerricAlphaError::ConflictingQuantileMode);
80        }
81        if let Some(quantiles) = self.quantiles {
82            if quantiles < 2 {
83                return Err(FerricAlphaError::InvalidQuantiles {
84                    reason: "quantiles must be at least 2",
85                });
86            }
87            if self.zero_aware && quantiles % 2 != 0 {
88                return Err(FerricAlphaError::InvalidQuantiles {
89                    reason: "zero-aware quantiles must be even",
90                });
91            }
92            return Ok(QuantizeMode::Quantiles(quantiles));
93        }
94
95        if let Some(edges) = self.quantile_edges.as_deref() {
96            if edges.len() < 2 {
97                return Err(FerricAlphaError::InvalidQuantiles {
98                    reason: "quantile edges must contain at least two boundaries",
99                });
100            }
101            for edge in edges {
102                if !edge.is_finite() || *edge < 0.0 || *edge > 1.0 {
103                    return Err(FerricAlphaError::InvalidQuantiles {
104                        reason: "quantile edges must be finite values in [0, 1]",
105                    });
106                }
107            }
108            for pair in edges.windows(2) {
109                if pair[0] >= pair[1] {
110                    return Err(FerricAlphaError::InvalidQuantiles {
111                        reason: "quantile edges must be strictly increasing",
112                    });
113                }
114            }
115            return Ok(QuantizeMode::QuantileEdges(edges));
116        }
117
118        if let Some(bin_count) = self.bin_count {
119            if bin_count < 1 {
120                return Err(FerricAlphaError::InvalidQuantiles {
121                    reason: "bins must be at least 1",
122                });
123            }
124            if self.zero_aware && bin_count % 2 != 0 {
125                return Err(FerricAlphaError::InvalidQuantiles {
126                    reason: "zero-aware bins must be even",
127                });
128            }
129            return Ok(QuantizeMode::BinCount(bin_count));
130        }
131
132        let bins = self
133            .bins
134            .as_deref()
135            .ok_or(FerricAlphaError::InvalidQuantiles {
136                reason: "quantiles or bins are required",
137            })?;
138        if bins.len() < 2 {
139            return Err(FerricAlphaError::InvalidQuantiles {
140                reason: "bins must contain at least two boundaries",
141            });
142        }
143        if self.zero_aware {
144            return Err(FerricAlphaError::InvalidQuantiles {
145                reason: "zero-aware explicit bins are not supported in this phase",
146            });
147        }
148        for pair in bins.windows(2) {
149            if !pair[0].is_finite() || !pair[1].is_finite() || pair[0] >= pair[1] {
150                return Err(FerricAlphaError::InvalidQuantiles {
151                    reason: "bin boundaries must be finite and strictly increasing",
152                });
153            }
154        }
155        Ok(QuantizeMode::Bins(bins))
156    }
157}
158
159#[derive(Debug, Clone, Copy)]
160enum QuantizeMode<'a> {
161    Quantiles(u32),
162    QuantileEdges(&'a [f64]),
163    BinCount(u32),
164    Bins(&'a [f64]),
165}
166
167pub fn quantize_factor(factor: &DataFrame, options: QuantizeOptions) -> Result<DataFrame> {
168    let mode = options.mode()?;
169    let factor = validate_factor_frame(factor)?;
170    if options.by_group && factor.column("group").is_err() {
171        return Err(FerricAlphaError::MissingColumn("group"));
172    }
173
174    let dates = factor.column("date")?.as_materialized_series().datetime()?;
175    let assets = factor.column("asset")?.as_materialized_series().str()?;
176    let factors = factor.column("factor")?.as_materialized_series().f64()?;
177    let groups = if options.by_group {
178        Some(factor.column("group")?.as_materialized_series().str()?)
179    } else {
180        None
181    };
182
183    let mut rows = Vec::with_capacity(factor.height());
184    let mut partitions: BTreeMap<(i64, Option<String>), Vec<usize>> = BTreeMap::new();
185    for index in 0..factor.height() {
186        let date = dates
187            .physical()
188            .get(index)
189            .expect("validated factor date should not be null");
190        let asset = assets
191            .get(index)
192            .expect("validated factor asset should not be null")
193            .to_owned();
194        let group = groups
195            .as_ref()
196            .and_then(|groups| groups.get(index).map(ToOwned::to_owned));
197        let factor_value = factors.get(index).filter(|value| value.is_finite());
198        rows.push(QuantileRow {
199            asset,
200            group: group.clone(),
201            factor: factor_value,
202        });
203        if factor_value.is_some() {
204            partitions.entry((date, group)).or_default().push(index);
205        }
206    }
207
208    let mut assignments = vec![None; rows.len()];
209    for indexes in partitions.values() {
210        match mode {
211            QuantizeMode::Quantiles(quantiles) if options.zero_aware => {
212                assign_zero_aware(indexes, &rows, quantiles, &mut assignments);
213            }
214            QuantizeMode::Quantiles(quantiles) => {
215                assign_quantiles(indexes, &rows, quantiles, 1, &mut assignments);
216            }
217            QuantizeMode::QuantileEdges(edges) => {
218                if options.zero_aware {
219                    assign_zero_aware_edges(indexes, &rows, edges, &mut assignments);
220                } else {
221                    assign_quantile_edges(indexes, &rows, edges, 1, &mut assignments);
222                }
223            }
224            QuantizeMode::BinCount(bin_count) if options.zero_aware => {
225                assign_zero_aware_bin_count(indexes, &rows, bin_count, &mut assignments);
226            }
227            QuantizeMode::BinCount(bin_count) => {
228                assign_bin_count(indexes, &rows, bin_count, 1, &mut assignments);
229            }
230            QuantizeMode::Bins(bins) => {
231                assign_bins(indexes, &rows, bins, false, &mut assignments);
232            }
233        }
234    }
235
236    let mut columns = factor.columns().to_vec();
237    columns.push(Series::new("factor_quantile".into(), assignments).into());
238    Ok(DataFrame::new(factor.height(), columns)?
239        .sort(["date", "asset"], SortMultipleOptions::default())?)
240}
241
242fn assign_bins(
243    indexes: &[usize],
244    rows: &[QuantileRow],
245    bins: &[f64],
246    include_lowest: bool,
247    assignments: &mut [Option<u32>],
248) {
249    for index in indexes {
250        let value = rows[*index].factor.expect("valid row should have factor");
251        let bucket = bins
252            .windows(2)
253            .enumerate()
254            .position(|(bucket, pair)| {
255                (value > pair[0] || (include_lowest && bucket == 0 && value == pair[0]))
256                    && value <= pair[1]
257            })
258            .map(|bucket| bucket as u32 + 1);
259        assignments[*index] = bucket;
260    }
261}
262
263fn assign_bin_count(
264    indexes: &[usize],
265    rows: &[QuantileRow],
266    bin_count: u32,
267    offset: u32,
268    assignments: &mut [Option<u32>],
269) {
270    if indexes.is_empty() {
271        return;
272    }
273    let values = indexes
274        .iter()
275        .map(|index| rows[*index].factor.expect("valid row should have factor"))
276        .collect::<Vec<_>>();
277    let min = values.iter().copied().reduce(f64::min).unwrap();
278    let max = values.iter().copied().reduce(f64::max).unwrap();
279    if bin_count == 1 || min == max {
280        for index in indexes {
281            assignments[*index] = Some(offset);
282        }
283        return;
284    }
285    let width = (max - min) / bin_count as f64;
286    for index in indexes {
287        let value = rows[*index].factor.expect("valid row should have factor");
288        let bucket = if value == min {
289            1
290        } else {
291            ((value - min) / width).ceil().clamp(1.0, bin_count as f64) as u32
292        };
293        assignments[*index] = Some(offset + bucket - 1);
294    }
295}
296
297#[derive(Debug)]
298struct QuantileRow {
299    asset: String,
300    group: Option<String>,
301    factor: Option<f64>,
302}
303
304fn assign_zero_aware(
305    indexes: &[usize],
306    rows: &[QuantileRow],
307    quantiles: u32,
308    assignments: &mut [Option<u32>],
309) {
310    let midpoint = quantiles / 2;
311    let negative = indexes
312        .iter()
313        .copied()
314        .filter(|index| rows[*index].factor.expect("valid row should have factor") < 0.0)
315        .collect::<Vec<_>>();
316    let positive = indexes
317        .iter()
318        .copied()
319        .filter(|index| rows[*index].factor.expect("valid row should have factor") >= 0.0)
320        .collect::<Vec<_>>();
321    assign_quantiles(&negative, rows, midpoint, 1, assignments);
322    assign_quantiles(&positive, rows, midpoint, midpoint + 1, assignments);
323}
324
325fn assign_zero_aware_edges(
326    indexes: &[usize],
327    rows: &[QuantileRow],
328    edges: &[f64],
329    assignments: &mut [Option<u32>],
330) {
331    let bucket_count = edges.len() as u32 - 1;
332    let midpoint = bucket_count / 2;
333    let negative = indexes
334        .iter()
335        .copied()
336        .filter(|index| rows[*index].factor.expect("valid row should have factor") < 0.0)
337        .collect::<Vec<_>>();
338    let positive = indexes
339        .iter()
340        .copied()
341        .filter(|index| rows[*index].factor.expect("valid row should have factor") >= 0.0)
342        .collect::<Vec<_>>();
343    assign_quantile_edges(&negative, rows, edges, 1, assignments);
344    assign_quantile_edges(&positive, rows, edges, midpoint + 1, assignments);
345}
346
347fn assign_zero_aware_bin_count(
348    indexes: &[usize],
349    rows: &[QuantileRow],
350    bin_count: u32,
351    assignments: &mut [Option<u32>],
352) {
353    let midpoint = bin_count / 2;
354    let negative = indexes
355        .iter()
356        .copied()
357        .filter(|index| rows[*index].factor.expect("valid row should have factor") < 0.0)
358        .collect::<Vec<_>>();
359    let positive = indexes
360        .iter()
361        .copied()
362        .filter(|index| rows[*index].factor.expect("valid row should have factor") >= 0.0)
363        .collect::<Vec<_>>();
364    assign_bin_count(&negative, rows, midpoint, 1, assignments);
365    assign_bin_count(&positive, rows, midpoint, midpoint + 1, assignments);
366}
367
368fn assign_quantile_edges(
369    indexes: &[usize],
370    rows: &[QuantileRow],
371    edges: &[f64],
372    offset: u32,
373    assignments: &mut [Option<u32>],
374) {
375    if indexes.is_empty() {
376        return;
377    }
378
379    let mut values = indexes
380        .iter()
381        .map(|index| rows[*index].factor.expect("valid row should have factor"))
382        .collect::<Vec<_>>();
383    values.sort_by(f64::total_cmp);
384    let cut_points = edges
385        .iter()
386        .map(|edge| percentile(&values, *edge))
387        .collect::<Vec<_>>();
388    assign_bins(indexes, rows, &cut_points, true, assignments);
389    for index in indexes {
390        if let Some(value) = &mut assignments[*index]
391            && *value > 0
392        {
393            *value += offset - 1;
394        }
395    }
396}
397
398fn assign_quantiles(
399    indexes: &[usize],
400    rows: &[QuantileRow],
401    quantiles: u32,
402    offset: u32,
403    assignments: &mut [Option<u32>],
404) {
405    if indexes.is_empty() {
406        return;
407    }
408    if indexes.len() < 2 {
409        return;
410    }
411
412    let mut sorted = indexes.to_vec();
413    sorted.sort_by(|left, right| {
414        let left_row = &rows[*left];
415        let right_row = &rows[*right];
416        left_row
417            .factor
418            .partial_cmp(&right_row.factor)
419            .expect("finite factors should be comparable")
420            .then_with(|| left_row.group.cmp(&right_row.group))
421            .then_with(|| left_row.asset.cmp(&right_row.asset))
422    });
423
424    for (rank, index) in sorted.iter().enumerate() {
425        let bucket = if sorted.len() >= quantiles as usize {
426            ((rank as u32 * quantiles) / sorted.len() as u32).min(quantiles - 1)
427        } else {
428            let scaled_rank = rank as f64 * (quantiles - 1) as f64 / (sorted.len() - 1) as f64;
429            scaled_rank.round() as u32
430        };
431        assignments[*index] = Some(offset + bucket);
432    }
433}
434
435fn percentile(sorted: &[f64], probability: f64) -> f64 {
436    if sorted.len() == 1 {
437        return sorted[0];
438    }
439    let position = probability * (sorted.len() - 1) as f64;
440    let lower = position.floor() as usize;
441    let upper = position.ceil() as usize;
442    if lower == upper {
443        sorted[lower]
444    } else {
445        let weight = position - lower as f64;
446        sorted[lower] * (1.0 - weight) + sorted[upper] * weight
447    }
448}