Skip to main content

ferric_alpha/
clean.rs

1use std::collections::HashMap;
2
3use polars::prelude::{DataFrame, NamedFrom, Series, SortMultipleOptions};
4
5use crate::calendar::{Period, validate_periods};
6use crate::data::{ensure_matching_date_timezone, validate_factor_frame};
7use crate::error::{FerricAlphaError, Result};
8use crate::forward_returns::{
9    ForwardReturnsOptions, compute_forward_returns_with_options, datetime_column,
10};
11use crate::quantiles::{QuantizeOptions, quantize_factor};
12
13type ReturnKey = (i64, String);
14type ReturnValues = Vec<Option<f64>>;
15
16#[derive(Debug, Clone, Copy, PartialEq)]
17pub struct CleanOptions {
18    pub max_loss: f64,
19}
20
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub struct LossReport {
23    pub input_rows: usize,
24    pub output_rows: usize,
25    pub forward_return_loss: usize,
26    pub quantile_loss: usize,
27}
28
29#[derive(Debug)]
30pub struct CleanFactorResult {
31    pub frame: DataFrame,
32    pub loss: LossReport,
33}
34
35pub fn get_clean_factor_and_forward_returns(
36    factor: &DataFrame,
37    prices: &DataFrame,
38    periods: &[Period],
39    quantize_options: QuantizeOptions,
40    clean_options: CleanOptions,
41) -> Result<CleanFactorResult> {
42    get_clean_factor_and_forward_returns_with_options(
43        factor,
44        prices,
45        periods,
46        quantize_options,
47        clean_options,
48        ForwardReturnsOptions::default(),
49    )
50}
51
52pub fn get_clean_factor_and_forward_returns_with_options(
53    factor: &DataFrame,
54    prices: &DataFrame,
55    periods: &[Period],
56    quantize_options: QuantizeOptions,
57    clean_options: CleanOptions,
58    forward_returns_options: ForwardReturnsOptions,
59) -> Result<CleanFactorResult> {
60    let periods = validate_periods(periods)?;
61    let returns =
62        compute_forward_returns_with_options(factor, prices, &periods, forward_returns_options)?;
63    get_clean_factor(factor, &returns, quantize_options, clean_options)
64}
65
66pub fn get_clean_factor(
67    factor: &DataFrame,
68    forward_returns: &DataFrame,
69    quantize_options: QuantizeOptions,
70    clean_options: CleanOptions,
71) -> Result<CleanFactorResult> {
72    let factor = validate_factor_frame(factor)?;
73    let date_dtype = ensure_matching_date_timezone(&factor, forward_returns)?;
74    let quantized = quantize_factor(&factor, quantize_options)?;
75    let return_columns = forward_return_columns(forward_returns);
76    if return_columns.is_empty() {
77        return Err(FerricAlphaError::MissingColumn("forward_return_*"));
78    }
79
80    let joined =
81        join_quantized_and_returns(&quantized, forward_returns, &return_columns, &date_dtype)?;
82    enforce_max_loss(&joined.loss, clean_options.max_loss)?;
83    Ok(joined)
84}
85
86fn forward_return_columns(frame: &DataFrame) -> Vec<String> {
87    let mut names = frame
88        .get_column_names()
89        .into_iter()
90        .map(ToString::to_string)
91        .filter(|name| name.starts_with("forward_return_"))
92        .collect::<Vec<_>>();
93    names.sort_by_key(|name| {
94        (
95            parse_forward_return_duration(name).unwrap_or(u64::MAX),
96            name.clone(),
97        )
98    });
99    names
100}
101
102fn parse_forward_return_duration(name: &str) -> Option<u64> {
103    let label = name.strip_prefix("forward_return_")?;
104    parse_duration_suffix(label)
105}
106
107fn parse_duration_suffix(label: &str) -> Option<u64> {
108    for (suffix, multiplier) in [
109        ("D", 86_400_000_000_u64),
110        ("ms", 1_000),
111        ("h", 3_600_000_000),
112        ("m", 60_000_000),
113        ("s", 1_000_000),
114    ] {
115        if let Some(value) = label.strip_suffix(suffix) {
116            return value.parse::<u64>().ok().map(|value| value * multiplier);
117        }
118    }
119    None
120}
121
122fn join_quantized_and_returns(
123    quantized: &DataFrame,
124    forward_returns: &DataFrame,
125    return_columns: &[String],
126    date_dtype: &polars::prelude::DataType,
127) -> Result<CleanFactorResult> {
128    let return_lookup = build_return_lookup(forward_returns, return_columns)?;
129    let dates = quantized
130        .column("date")?
131        .as_materialized_series()
132        .datetime()?;
133    let assets = quantized.column("asset")?.as_materialized_series().str()?;
134    let factors = quantized.column("factor")?.as_materialized_series().f64()?;
135    let quantiles = quantized
136        .column("factor_quantile")?
137        .as_materialized_series()
138        .u32()?;
139    let groups = quantized
140        .column("group")
141        .ok()
142        .map(|column| column.as_materialized_series().str())
143        .transpose()?;
144
145    let mut output_dates = Vec::new();
146    let mut output_assets = Vec::new();
147    let mut output_factors = Vec::new();
148    let mut output_groups = Vec::new();
149    let mut output_quantiles = Vec::new();
150    let mut output_returns = vec![Vec::new(); return_columns.len()];
151    let mut forward_return_loss = 0;
152    let mut quantile_loss = 0;
153
154    for index in 0..quantized.height() {
155        let date = dates
156            .physical()
157            .get(index)
158            .expect("validated date should not be null");
159        let asset = assets
160            .get(index)
161            .expect("validated asset should not be null");
162        let factor = factors.get(index);
163        let quantile = quantiles.get(index);
164        let returns = return_lookup
165            .get(&(date, asset.to_owned()))
166            .cloned()
167            .unwrap_or_else(|| vec![None; return_columns.len()]);
168
169        let row_return_loss = returns.iter().any(Option::is_none);
170        let row_quantile_loss = quantile.is_none();
171        forward_return_loss += usize::from(row_return_loss);
172        quantile_loss += usize::from(row_quantile_loss);
173
174        if row_return_loss || row_quantile_loss {
175            continue;
176        }
177
178        output_dates.push(date);
179        output_assets.push(asset.to_owned());
180        output_factors.push(factor);
181        if let Some(groups) = groups {
182            output_groups.push(groups.get(index).map(ToOwned::to_owned));
183        }
184        output_quantiles.push(quantile);
185        for (return_index, value) in returns.into_iter().enumerate() {
186            output_returns[return_index].push(value);
187        }
188    }
189
190    let height = output_dates.len();
191    let mut columns = vec![
192        datetime_column("date", output_dates, date_dtype)?,
193        Series::new("asset".into(), output_assets).into(),
194        Series::new("factor".into(), output_factors).into(),
195    ];
196    if groups.is_some() {
197        columns.push(Series::new("group".into(), output_groups).into());
198    }
199    columns.push(Series::new("factor_quantile".into(), output_quantiles).into());
200    for (name, values) in return_columns.iter().zip(output_returns) {
201        columns.push(Series::new(name.clone().into(), values).into());
202    }
203
204    let frame =
205        DataFrame::new(height, columns)?.sort(["date", "asset"], SortMultipleOptions::default())?;
206    Ok(CleanFactorResult {
207        frame,
208        loss: LossReport {
209            input_rows: quantized.height(),
210            output_rows: height,
211            forward_return_loss,
212            quantile_loss,
213        },
214    })
215}
216
217fn build_return_lookup(
218    forward_returns: &DataFrame,
219    return_columns: &[String],
220) -> Result<HashMap<ReturnKey, ReturnValues>> {
221    let dates = forward_returns
222        .column("date")?
223        .as_materialized_series()
224        .datetime()?;
225    let assets = forward_returns
226        .column("asset")?
227        .as_materialized_series()
228        .str()?;
229    let return_series = return_columns
230        .iter()
231        .map(|name| {
232            Ok(forward_returns
233                .column(name)?
234                .as_materialized_series()
235                .f64()?)
236        })
237        .collect::<Result<Vec<_>>>()?;
238    let mut lookup = HashMap::with_capacity(forward_returns.height());
239    for index in 0..forward_returns.height() {
240        let date = dates
241            .physical()
242            .get(index)
243            .ok_or(FerricAlphaError::NullKey)?;
244        let asset = assets.get(index).ok_or(FerricAlphaError::NullKey)?;
245        let values = return_series
246            .iter()
247            .map(|series| series.get(index).filter(|value| value.is_finite()))
248            .collect::<Vec<_>>();
249        if lookup.insert((date, asset.to_owned()), values).is_some() {
250            return Err(FerricAlphaError::DuplicateKey);
251        }
252    }
253    Ok(lookup)
254}
255
256fn enforce_max_loss(loss: &LossReport, max_loss: f64) -> Result<()> {
257    if !max_loss.is_finite() || !(0.0..=1.0).contains(&max_loss) {
258        return Err(FerricAlphaError::InvalidMaxLoss { value: max_loss });
259    }
260
261    let actual = if loss.input_rows == 0 {
262        0.0
263    } else {
264        (loss.input_rows - loss.output_rows) as f64 / loss.input_rows as f64
265    };
266    if actual > max_loss {
267        return Err(FerricAlphaError::MaxLossExceeded {
268            actual,
269            allowed: max_loss,
270        });
271    }
272
273    Ok(())
274}