1use std::collections::HashMap;
2
3use polars::prelude::{DataFrame, NamedFrom, Series, SortMultipleOptions};
4
5use crate::calendar::{Period, validate_periods};
6use crate::data::{ensure_matching_date_timezone, validate_factor_frame};
7use crate::error::{FerricAlphaError, Result};
8use crate::forward_returns::{
9 ForwardReturnsOptions, compute_forward_returns_with_options, datetime_column,
10};
11use crate::quantiles::{QuantizeOptions, quantize_factor};
12
13type ReturnKey = (i64, String);
14type ReturnValues = Vec<Option<f64>>;
15
16#[derive(Debug, Clone, Copy, PartialEq)]
17pub struct CleanOptions {
18 pub max_loss: f64,
19}
20
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub struct LossReport {
23 pub input_rows: usize,
24 pub output_rows: usize,
25 pub forward_return_loss: usize,
26 pub quantile_loss: usize,
27}
28
29#[derive(Debug)]
30pub struct CleanFactorResult {
31 pub frame: DataFrame,
32 pub loss: LossReport,
33}
34
35pub fn get_clean_factor_and_forward_returns(
36 factor: &DataFrame,
37 prices: &DataFrame,
38 periods: &[Period],
39 quantize_options: QuantizeOptions,
40 clean_options: CleanOptions,
41) -> Result<CleanFactorResult> {
42 get_clean_factor_and_forward_returns_with_options(
43 factor,
44 prices,
45 periods,
46 quantize_options,
47 clean_options,
48 ForwardReturnsOptions::default(),
49 )
50}
51
52pub fn get_clean_factor_and_forward_returns_with_options(
53 factor: &DataFrame,
54 prices: &DataFrame,
55 periods: &[Period],
56 quantize_options: QuantizeOptions,
57 clean_options: CleanOptions,
58 forward_returns_options: ForwardReturnsOptions,
59) -> Result<CleanFactorResult> {
60 let periods = validate_periods(periods)?;
61 let returns =
62 compute_forward_returns_with_options(factor, prices, &periods, forward_returns_options)?;
63 get_clean_factor(factor, &returns, quantize_options, clean_options)
64}
65
66pub fn get_clean_factor(
67 factor: &DataFrame,
68 forward_returns: &DataFrame,
69 quantize_options: QuantizeOptions,
70 clean_options: CleanOptions,
71) -> Result<CleanFactorResult> {
72 let factor = validate_factor_frame(factor)?;
73 let date_dtype = ensure_matching_date_timezone(&factor, forward_returns)?;
74 let quantized = quantize_factor(&factor, quantize_options)?;
75 let return_columns = forward_return_columns(forward_returns);
76 if return_columns.is_empty() {
77 return Err(FerricAlphaError::MissingColumn("forward_return_*"));
78 }
79
80 let joined =
81 join_quantized_and_returns(&quantized, forward_returns, &return_columns, &date_dtype)?;
82 enforce_max_loss(&joined.loss, clean_options.max_loss)?;
83 Ok(joined)
84}
85
86fn forward_return_columns(frame: &DataFrame) -> Vec<String> {
87 let mut names = frame
88 .get_column_names()
89 .into_iter()
90 .map(ToString::to_string)
91 .filter(|name| name.starts_with("forward_return_"))
92 .collect::<Vec<_>>();
93 names.sort_by_key(|name| {
94 (
95 parse_forward_return_duration(name).unwrap_or(u64::MAX),
96 name.clone(),
97 )
98 });
99 names
100}
101
102fn parse_forward_return_duration(name: &str) -> Option<u64> {
103 let label = name.strip_prefix("forward_return_")?;
104 parse_duration_suffix(label)
105}
106
107fn parse_duration_suffix(label: &str) -> Option<u64> {
108 for (suffix, multiplier) in [
109 ("D", 86_400_000_000_u64),
110 ("ms", 1_000),
111 ("h", 3_600_000_000),
112 ("m", 60_000_000),
113 ("s", 1_000_000),
114 ] {
115 if let Some(value) = label.strip_suffix(suffix) {
116 return value.parse::<u64>().ok().map(|value| value * multiplier);
117 }
118 }
119 None
120}
121
122fn join_quantized_and_returns(
123 quantized: &DataFrame,
124 forward_returns: &DataFrame,
125 return_columns: &[String],
126 date_dtype: &polars::prelude::DataType,
127) -> Result<CleanFactorResult> {
128 let return_lookup = build_return_lookup(forward_returns, return_columns)?;
129 let dates = quantized
130 .column("date")?
131 .as_materialized_series()
132 .datetime()?;
133 let assets = quantized.column("asset")?.as_materialized_series().str()?;
134 let factors = quantized.column("factor")?.as_materialized_series().f64()?;
135 let quantiles = quantized
136 .column("factor_quantile")?
137 .as_materialized_series()
138 .u32()?;
139 let groups = quantized
140 .column("group")
141 .ok()
142 .map(|column| column.as_materialized_series().str())
143 .transpose()?;
144
145 let mut output_dates = Vec::new();
146 let mut output_assets = Vec::new();
147 let mut output_factors = Vec::new();
148 let mut output_groups = Vec::new();
149 let mut output_quantiles = Vec::new();
150 let mut output_returns = vec![Vec::new(); return_columns.len()];
151 let mut forward_return_loss = 0;
152 let mut quantile_loss = 0;
153
154 for index in 0..quantized.height() {
155 let date = dates
156 .physical()
157 .get(index)
158 .expect("validated date should not be null");
159 let asset = assets
160 .get(index)
161 .expect("validated asset should not be null");
162 let factor = factors.get(index);
163 let quantile = quantiles.get(index);
164 let returns = return_lookup
165 .get(&(date, asset.to_owned()))
166 .cloned()
167 .unwrap_or_else(|| vec![None; return_columns.len()]);
168
169 let row_return_loss = returns.iter().any(Option::is_none);
170 let row_quantile_loss = quantile.is_none();
171 forward_return_loss += usize::from(row_return_loss);
172 quantile_loss += usize::from(row_quantile_loss);
173
174 if row_return_loss || row_quantile_loss {
175 continue;
176 }
177
178 output_dates.push(date);
179 output_assets.push(asset.to_owned());
180 output_factors.push(factor);
181 if let Some(groups) = groups {
182 output_groups.push(groups.get(index).map(ToOwned::to_owned));
183 }
184 output_quantiles.push(quantile);
185 for (return_index, value) in returns.into_iter().enumerate() {
186 output_returns[return_index].push(value);
187 }
188 }
189
190 let height = output_dates.len();
191 let mut columns = vec![
192 datetime_column("date", output_dates, date_dtype)?,
193 Series::new("asset".into(), output_assets).into(),
194 Series::new("factor".into(), output_factors).into(),
195 ];
196 if groups.is_some() {
197 columns.push(Series::new("group".into(), output_groups).into());
198 }
199 columns.push(Series::new("factor_quantile".into(), output_quantiles).into());
200 for (name, values) in return_columns.iter().zip(output_returns) {
201 columns.push(Series::new(name.clone().into(), values).into());
202 }
203
204 let frame =
205 DataFrame::new(height, columns)?.sort(["date", "asset"], SortMultipleOptions::default())?;
206 Ok(CleanFactorResult {
207 frame,
208 loss: LossReport {
209 input_rows: quantized.height(),
210 output_rows: height,
211 forward_return_loss,
212 quantile_loss,
213 },
214 })
215}
216
217fn build_return_lookup(
218 forward_returns: &DataFrame,
219 return_columns: &[String],
220) -> Result<HashMap<ReturnKey, ReturnValues>> {
221 let dates = forward_returns
222 .column("date")?
223 .as_materialized_series()
224 .datetime()?;
225 let assets = forward_returns
226 .column("asset")?
227 .as_materialized_series()
228 .str()?;
229 let return_series = return_columns
230 .iter()
231 .map(|name| {
232 Ok(forward_returns
233 .column(name)?
234 .as_materialized_series()
235 .f64()?)
236 })
237 .collect::<Result<Vec<_>>>()?;
238 let mut lookup = HashMap::with_capacity(forward_returns.height());
239 for index in 0..forward_returns.height() {
240 let date = dates
241 .physical()
242 .get(index)
243 .ok_or(FerricAlphaError::NullKey)?;
244 let asset = assets.get(index).ok_or(FerricAlphaError::NullKey)?;
245 let values = return_series
246 .iter()
247 .map(|series| series.get(index).filter(|value| value.is_finite()))
248 .collect::<Vec<_>>();
249 if lookup.insert((date, asset.to_owned()), values).is_some() {
250 return Err(FerricAlphaError::DuplicateKey);
251 }
252 }
253 Ok(lookup)
254}
255
256fn enforce_max_loss(loss: &LossReport, max_loss: f64) -> Result<()> {
257 if !max_loss.is_finite() || !(0.0..=1.0).contains(&max_loss) {
258 return Err(FerricAlphaError::InvalidMaxLoss { value: max_loss });
259 }
260
261 let actual = if loss.input_rows == 0 {
262 0.0
263 } else {
264 (loss.input_rows - loss.output_rows) as f64 / loss.input_rows as f64
265 };
266 if actual > max_loss {
267 return Err(FerricAlphaError::MaxLossExceeded {
268 actual,
269 allowed: max_loss,
270 });
271 }
272
273 Ok(())
274}