1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
2
3use polars::prelude::{DataFrame, DataType, IntoColumn, NamedFrom, Series, SortMultipleOptions};
4use serde::{Deserialize, Serialize};
5
6use crate::data::{ensure_matching_date_timezone, required_column};
7use crate::error::{FerricAlphaError, Result};
8use crate::performance::frame::{ensure_dtype, validate_factor_data};
9use crate::statistics::sample_std;
10
11const OPERATION: &str = "average_cumulative_return_by_quantile";
12
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
14pub struct EventReturnsOptions {
15 pub periods_before: u32,
16 pub periods_after: u32,
17 pub demeaned: bool,
18 pub group_adjust: bool,
19 pub by_group: bool,
20}
21
22impl Default for EventReturnsOptions {
23 fn default() -> Self {
24 Self {
25 periods_before: 10,
26 periods_after: 15,
27 demeaned: true,
28 group_adjust: false,
29 by_group: false,
30 }
31 }
32}
33
34impl EventReturnsOptions {
35 pub fn with_periods_before(mut self, periods_before: u32) -> Self {
36 self.periods_before = periods_before;
37 self
38 }
39
40 pub fn with_periods_after(mut self, periods_after: u32) -> Self {
41 self.periods_after = periods_after;
42 self
43 }
44
45 pub fn with_demeaned(mut self, demeaned: bool) -> Self {
46 self.demeaned = demeaned;
47 self
48 }
49
50 pub fn with_group_adjust(mut self, group_adjust: bool) -> Self {
51 self.group_adjust = group_adjust;
52 self
53 }
54
55 pub fn with_by_group(mut self, by_group: bool) -> Self {
56 self.by_group = by_group;
57 self
58 }
59}
60
61#[derive(Debug, Clone, PartialEq, Eq)]
62#[allow(dead_code)]
63pub(crate) struct EventCoverageRow {
64 pub(crate) scope: String,
65 pub(crate) offset: Option<i32>,
66 pub(crate) input_events: u64,
67 pub(crate) used_events: u64,
68 pub(crate) skipped_missing_date: u64,
69 pub(crate) imputed_return_cells: u64,
70 pub(crate) path_count: u64,
71}
72
73#[derive(Debug, Clone)]
74pub(crate) struct EventWindowResult {
75 pub(crate) frame: DataFrame,
76 #[allow(dead_code)]
77 pub(crate) coverage: Vec<EventCoverageRow>,
78}
79
80#[derive(Debug, Clone)]
81struct Event {
82 date: i64,
83 session_index: usize,
84 asset: String,
85 group: Option<String>,
86 quantile: u32,
87}
88
89#[derive(Debug, Clone)]
90struct ReturnCalendar {
91 dates: Vec<i64>,
92 values: HashMap<(String, i64), Option<f64>>,
93}
94
95#[derive(Debug, Clone, Default)]
96struct OffsetCoverage {
97 used_events: u64,
98 imputed_return_cells: u64,
99 path_count: u64,
100}
101
102pub fn average_cumulative_return_by_quantile(
103 factor_data: &DataFrame,
104 returns: &DataFrame,
105 options: EventReturnsOptions,
106) -> Result<DataFrame> {
107 Ok(event_window_result(factor_data, returns, options)?.frame)
108}
109
110pub(crate) fn event_window_result(
111 factor_data: &DataFrame,
112 returns: &DataFrame,
113 options: EventReturnsOptions,
114) -> Result<EventWindowResult> {
115 let data = validate_factor_data(
116 factor_data,
117 false,
118 options.group_adjust || options.by_group,
119 false,
120 )?;
121 let quantile = required_column(factor_data, "factor_quantile")?;
122 ensure_dtype("factor_quantile", quantile.dtype(), "UInt32", |dtype| {
123 matches!(dtype, DataType::UInt32)
124 })?;
125
126 let calendar = validate_returns(factor_data, returns)?;
127 if calendar.dates.is_empty() {
128 return Err(FerricAlphaError::InsufficientData {
129 operation: OPERATION,
130 });
131 }
132 let date_index = calendar
133 .dates
134 .iter()
135 .enumerate()
136 .map(|(index, date)| (*date, index))
137 .collect::<HashMap<_, _>>();
138
139 let mut input_events = 0_u64;
140 let mut skipped_missing_date = 0_u64;
141 let mut events = Vec::new();
142 for row in &data.rows {
143 let Some(quantile) = row.quantile.filter(|quantile| *quantile > 0) else {
144 continue;
145 };
146 if (options.group_adjust || options.by_group) && row.group.is_none() {
147 return Err(FerricAlphaError::InvalidInput {
148 operation: OPERATION,
149 reason: "group must not contain null values",
150 });
151 }
152 input_events += 1;
153 let Some(session_index) = date_index.get(&row.date).copied() else {
154 skipped_missing_date += 1;
155 continue;
156 };
157 events.push(Event {
158 date: row.date,
159 session_index,
160 asset: row.asset.clone(),
161 group: row.group.clone(),
162 quantile,
163 });
164 }
165
166 if events.is_empty() {
167 return Err(FerricAlphaError::InsufficientData {
168 operation: OPERATION,
169 });
170 }
171
172 let event_dates = events
173 .iter()
174 .map(|event| event.date)
175 .collect::<BTreeSet<_>>();
176 let mut assets = events
177 .iter()
178 .map(|event| event.asset.clone())
179 .collect::<BTreeSet<_>>();
180 if options.demeaned || options.group_adjust {
181 for row in &data.rows {
182 if event_dates.contains(&row.date) {
183 assets.insert(row.asset.clone());
184 }
185 }
186 }
187
188 let (wealth, densify_imputations) = cumulative_wealth(&calendar, &assets);
189 let by_event_date = group_events_by_date_partition(&events, options.by_group);
190 let universe_by_date = universe_by_date(&data.rows, &event_dates);
191 let universe_by_date_group = universe_by_date_group(&data.rows, &event_dates);
192
193 let mut paths: BTreeMap<(u32, Option<String>, i32), Vec<f64>> = BTreeMap::new();
194 let mut coverage_by_offset: BTreeMap<i32, OffsetCoverage> = BTreeMap::new();
195 let start = -(options.periods_before as i32);
196 let end = options.periods_after as i32;
197 for event_offset in start..=end {
198 coverage_by_offset.entry(event_offset).or_default();
199 }
200
201 for ((event_date, quantile, group), partition_events) in by_event_date {
202 let session_index = partition_events[0].session_index;
203 for offset in start..=end {
204 let target_index = session_index as i32 + offset;
205 if target_index < 0 || target_index >= calendar.dates.len() as i32 {
206 continue;
207 }
208 let target_index = target_index as usize;
209 let mut values = Vec::with_capacity(partition_events.len());
210 let mut imputed = 0_u64;
211 for event in &partition_events {
212 let (mut value, was_imputed) =
213 rebased_wealth_value(&wealth, &event.asset, event.session_index, target_index);
214 if options.group_adjust {
215 let group_assets = event
216 .group
217 .as_ref()
218 .and_then(|group| universe_by_date_group.get(&(event_date, group.clone())));
219 let (group_mean, group_imputed) = mean_rebased_wealth_with_imputations(
220 group_assets,
221 &wealth,
222 session_index,
223 target_index,
224 );
225 value -= group_mean.unwrap_or(0.0);
226 imputed += group_imputed;
227 }
228 values.push(value);
229 imputed += u64::from(was_imputed);
230 }
231 let Some(mut event_path) = mean(&values) else {
232 continue;
233 };
234
235 if options.demeaned && !options.group_adjust {
236 let (universe_mean, universe_imputed) = mean_rebased_wealth_with_imputations(
237 universe_by_date.get(&event_date),
238 &wealth,
239 session_index,
240 target_index,
241 );
242 event_path -= universe_mean.unwrap_or(0.0);
243 imputed += universe_imputed;
244 }
245
246 let key = (quantile, group.clone().filter(|_| options.by_group), offset);
247 paths.entry(key).or_default().push(event_path);
248 let offset_coverage = coverage_by_offset.entry(offset).or_default();
249 offset_coverage.used_events += partition_events.len() as u64;
250 offset_coverage.imputed_return_cells += imputed;
251 offset_coverage.path_count += 1;
252 }
253 }
254
255 let mut factor_quantiles = Vec::new();
256 let mut groups = Vec::new();
257 let mut offsets = Vec::new();
258 let mut means = Vec::new();
259 let mut stds = Vec::new();
260 let mut counts = Vec::new();
261 for ((quantile, group, offset), values) in paths {
262 factor_quantiles.push(quantile);
263 if options.by_group {
264 groups.push(group.expect("grouped output should carry a group"));
265 }
266 offsets.push(offset);
267 means.push(mean(&values));
268 stds.push(sample_std(&values)?);
269 counts.push(values.len() as u64);
270 }
271
272 let mut columns = vec![
273 Series::new("factor_quantile".into(), factor_quantiles).into_column(),
274 Series::new("offset".into(), offsets).into_column(),
275 Series::new("mean_cumulative_return".into(), means).into_column(),
276 Series::new("std_cumulative_return".into(), stds).into_column(),
277 Series::new("count".into(), counts).into_column(),
278 ];
279 let sort = if options.by_group {
280 columns.insert(1, Series::new("group".into(), groups).into_column());
281 vec!["factor_quantile", "group", "offset"]
282 } else {
283 vec!["factor_quantile", "offset"]
284 };
285 let frame =
286 DataFrame::new(columns[0].len(), columns)?.sort(sort, SortMultipleOptions::default())?;
287
288 let mut coverage = Vec::with_capacity(coverage_by_offset.len() + 1);
289 coverage.push(EventCoverageRow {
290 scope: "overall".to_owned(),
291 offset: None,
292 input_events,
293 used_events: events.len() as u64,
294 skipped_missing_date,
295 imputed_return_cells: densify_imputations,
296 path_count: 0,
297 });
298 for (offset, row) in coverage_by_offset {
299 coverage.push(EventCoverageRow {
300 scope: "offset".to_owned(),
301 offset: Some(offset),
302 input_events,
303 used_events: row.used_events,
304 skipped_missing_date,
305 imputed_return_cells: row.imputed_return_cells,
306 path_count: row.path_count,
307 });
308 }
309
310 Ok(EventWindowResult { frame, coverage })
311}
312
313fn validate_returns(factor_data: &DataFrame, returns: &DataFrame) -> Result<ReturnCalendar> {
314 ensure_matching_date_timezone(factor_data, returns)?;
315 let date = required_column(returns, "date")?;
316 let asset = required_column(returns, "asset")?;
317 let ret = required_column(returns, "return")?;
318 ensure_dtype("asset", asset.dtype(), "String", |dtype| {
319 matches!(dtype, DataType::String)
320 })?;
321 ensure_dtype("return", ret.dtype(), "Float64", |dtype| {
322 matches!(dtype, DataType::Float64)
323 })?;
324 if date.null_count() > 0 || asset.null_count() > 0 {
325 return Err(FerricAlphaError::NullKey);
326 }
327
328 let dates = date.as_materialized_series().datetime()?;
329 let assets = asset.as_materialized_series().str()?;
330 let returns_values = ret.as_materialized_series().f64()?;
331 let mut seen = HashSet::with_capacity(returns.height());
332 let mut calendar_dates = BTreeSet::new();
333 let mut values = HashMap::with_capacity(returns.height());
334 for ((date, asset), ret) in dates
335 .physical()
336 .iter()
337 .zip(assets.iter())
338 .zip(returns_values.iter())
339 {
340 let (Some(date), Some(asset)) = (date, asset) else {
341 return Err(FerricAlphaError::NullKey);
342 };
343 if !seen.insert((date, asset.to_owned())) {
344 return Err(FerricAlphaError::DuplicateKey);
345 }
346 calendar_dates.insert(date);
347 values.insert(
348 (asset.to_owned(), date),
349 ret.filter(|value| value.is_finite()),
350 );
351 }
352
353 Ok(ReturnCalendar {
354 dates: calendar_dates.into_iter().collect(),
355 values,
356 })
357}
358
359fn cumulative_wealth(
360 calendar: &ReturnCalendar,
361 assets: &BTreeSet<String>,
362) -> (HashMap<String, Vec<(f64, bool)>>, u64) {
363 let mut wealth = HashMap::with_capacity(assets.len());
364 let mut imputed = 0_u64;
365 for asset in assets {
366 let mut cumulative = 1.0;
367 let mut values = Vec::with_capacity(calendar.dates.len());
368 for date in &calendar.dates {
369 let return_value = calendar
370 .values
371 .get(&(asset.clone(), *date))
372 .copied()
373 .flatten();
374 let was_imputed = return_value.is_none();
375 if was_imputed {
376 imputed += 1;
377 }
378 cumulative *= 1.0 + return_value.unwrap_or(0.0);
379 values.push((cumulative - 1.0, was_imputed));
380 }
381 wealth.insert(asset.clone(), values);
382 }
383 (wealth, imputed)
384}
385
386fn group_events_by_date_partition(
387 events: &[Event],
388 by_group: bool,
389) -> BTreeMap<(i64, u32, Option<String>), Vec<Event>> {
390 let mut grouped = BTreeMap::new();
391 for event in events {
392 grouped
393 .entry((
394 event.date,
395 event.quantile,
396 event.group.clone().filter(|_| by_group),
397 ))
398 .or_insert_with(Vec::new)
399 .push(event.clone());
400 }
401 grouped
402}
403
404fn universe_by_date(
405 rows: &[crate::performance::frame::FactorRow],
406 event_dates: &BTreeSet<i64>,
407) -> BTreeMap<i64, Vec<String>> {
408 let mut universes: BTreeMap<i64, BTreeSet<String>> = BTreeMap::new();
409 for row in rows {
410 if event_dates.contains(&row.date) {
411 universes
412 .entry(row.date)
413 .or_default()
414 .insert(row.asset.clone());
415 }
416 }
417 universes
418 .into_iter()
419 .map(|(date, assets)| (date, assets.into_iter().collect()))
420 .collect()
421}
422
423fn universe_by_date_group(
424 rows: &[crate::performance::frame::FactorRow],
425 event_dates: &BTreeSet<i64>,
426) -> BTreeMap<(i64, String), Vec<String>> {
427 let mut universes: BTreeMap<(i64, String), BTreeSet<String>> = BTreeMap::new();
428 for row in rows {
429 if event_dates.contains(&row.date)
430 && let Some(group) = &row.group
431 {
432 universes
433 .entry((row.date, group.clone()))
434 .or_default()
435 .insert(row.asset.clone());
436 }
437 }
438 universes
439 .into_iter()
440 .map(|(key, assets)| (key, assets.into_iter().collect()))
441 .collect()
442}
443
444fn mean_rebased_wealth_with_imputations(
445 assets: Option<&Vec<String>>,
446 wealth: &HashMap<String, Vec<(f64, bool)>>,
447 base_index: usize,
448 target_index: usize,
449) -> (Option<f64>, u64) {
450 let Some(assets) = assets else {
451 return (None, 0);
452 };
453 let mut imputed = 0_u64;
454 let values = assets
455 .iter()
456 .map(|asset| {
457 let (value, was_imputed) =
458 rebased_wealth_value(wealth, asset, base_index, target_index);
459 imputed += u64::from(was_imputed);
460 value
461 })
462 .collect::<Vec<_>>();
463 (mean(&values), imputed)
464}
465
466fn wealth_value(
467 wealth: &HashMap<String, Vec<(f64, bool)>>,
468 asset: &str,
469 target_index: usize,
470) -> (f64, bool) {
471 wealth
472 .get(asset)
473 .map(|path| path[target_index])
474 .unwrap_or((0.0, true))
475}
476
477fn rebased_wealth_value(
478 wealth: &HashMap<String, Vec<(f64, bool)>>,
479 asset: &str,
480 base_index: usize,
481 target_index: usize,
482) -> (f64, bool) {
483 let (target, was_imputed) = wealth_value(wealth, asset, target_index);
484 let (base, _) = wealth_value(wealth, asset, base_index);
485 (((1.0 + target) / (1.0 + base)) - 1.0, was_imputed)
486}
487
488fn mean(values: &[f64]) -> Option<f64> {
489 if values.is_empty() {
490 None
491 } else {
492 Some(values.iter().sum::<f64>() / values.len() as f64)
493 }
494}
495
496#[cfg(test)]
497mod tests {
498 use polars::prelude::{DataFrame, DataType, IntoColumn, NamedFrom, Series, TimeUnit};
499
500 use super::{EventReturnsOptions, event_window_result};
501
502 const JAN_1: i64 = 1_704_067_200_000;
503 const DAY_MS: i64 = 86_400_000;
504
505 fn datetime_column(values: &[i64]) -> polars::prelude::Column {
506 Series::new("date".into(), values)
507 .cast(&DataType::Datetime(TimeUnit::Milliseconds, None))
508 .expect("test dates should cast")
509 .into_column()
510 }
511
512 #[test]
513 fn event_window_result_carries_overall_and_offset_coverage() {
514 let jan_2 = JAN_1 + DAY_MS;
515 let jan_3 = jan_2 + DAY_MS;
516 let absent = jan_3 + DAY_MS;
517 let factor_data = DataFrame::new(
518 3,
519 vec![
520 datetime_column(&[JAN_1, jan_2, absent]),
521 Series::new("asset".into(), &["A", "A", "A"]).into_column(),
522 Series::new("factor".into(), &[Some(1.0); 3]).into_column(),
523 Series::new("factor_quantile".into(), &[Some(1_u32); 3]).into_column(),
524 ],
525 )
526 .expect("test factor data should build");
527 let returns = DataFrame::new(
528 3,
529 vec![
530 datetime_column(&[JAN_1, jan_2, jan_3]),
531 Series::new("asset".into(), &["A", "A", "A"]).into_column(),
532 Series::new("return".into(), &[Some(0.10), None, Some(0.10)]).into_column(),
533 ],
534 )
535 .expect("test returns should build");
536
537 let result = event_window_result(
538 &factor_data,
539 &returns,
540 EventReturnsOptions {
541 periods_before: 1,
542 periods_after: 1,
543 demeaned: false,
544 group_adjust: false,
545 by_group: false,
546 },
547 )
548 .unwrap();
549
550 assert_eq!(result.coverage[0].scope, "overall");
551 assert_eq!(result.coverage[0].offset, None);
552 assert_eq!(result.coverage[0].input_events, 3);
553 assert_eq!(result.coverage[0].used_events, 2);
554 assert_eq!(result.coverage[0].skipped_missing_date, 1);
555 assert_eq!(result.coverage[0].imputed_return_cells, 1);
556
557 let offset_zero = result
558 .coverage
559 .iter()
560 .find(|row| row.offset == Some(0))
561 .expect("offset zero coverage should exist");
562 assert_eq!(offset_zero.scope, "offset");
563 assert_eq!(offset_zero.used_events, 2);
564 assert_eq!(offset_zero.imputed_return_cells, 1);
565 assert_eq!(offset_zero.path_count, 2);
566 }
567
568 #[test]
569 fn event_window_coverage_counts_raw_events_and_adjustment_imputations() {
570 let factor_data = DataFrame::new(
571 2,
572 vec![
573 datetime_column(&[JAN_1, JAN_1]),
574 Series::new("asset".into(), &["A", "B"]).into_column(),
575 Series::new("group".into(), &["g1", "g1"]).into_column(),
576 Series::new("factor".into(), &[Some(1.0); 2]).into_column(),
577 Series::new("factor_quantile".into(), &[Some(1_u32); 2]).into_column(),
578 ],
579 )
580 .expect("test factor data should build");
581 let returns = DataFrame::new(
582 2,
583 vec![
584 datetime_column(&[JAN_1, JAN_1]),
585 Series::new("asset".into(), &["A", "B"]).into_column(),
586 Series::new("return".into(), &[Some(0.10), None]).into_column(),
587 ],
588 )
589 .expect("test returns should build");
590
591 let result = event_window_result(
592 &factor_data,
593 &returns,
594 EventReturnsOptions {
595 periods_before: 0,
596 periods_after: 0,
597 demeaned: false,
598 group_adjust: true,
599 by_group: false,
600 },
601 )
602 .unwrap();
603
604 let offset_zero = result
605 .coverage
606 .iter()
607 .find(|row| row.offset == Some(0))
608 .expect("offset zero coverage should exist");
609 assert_eq!(offset_zero.used_events, 2);
610 assert_eq!(offset_zero.imputed_return_cells, 3);
611 assert_eq!(offset_zero.path_count, 1);
612 }
613
614 #[test]
615 fn event_window_rebases_each_path_to_the_event_session() {
616 let jan_2 = JAN_1 + DAY_MS;
617 let jan_3 = jan_2 + DAY_MS;
618 let factor_data = DataFrame::new(
619 1,
620 vec![
621 datetime_column(&[jan_2]),
622 Series::new("asset".into(), &["A"]).into_column(),
623 Series::new("factor".into(), &[Some(1.0)]).into_column(),
624 Series::new("factor_quantile".into(), &[Some(1_u32)]).into_column(),
625 ],
626 )
627 .expect("test factor data should build");
628 let returns = DataFrame::new(
629 3,
630 vec![
631 datetime_column(&[JAN_1, jan_2, jan_3]),
632 Series::new("asset".into(), &["A", "A", "A"]).into_column(),
633 Series::new("return".into(), &[Some(0.10), Some(0.10), Some(0.10)]).into_column(),
634 ],
635 )
636 .expect("test returns should build");
637
638 let result = event_window_result(
639 &factor_data,
640 &returns,
641 EventReturnsOptions {
642 periods_before: 1,
643 periods_after: 1,
644 demeaned: false,
645 group_adjust: false,
646 by_group: false,
647 },
648 )
649 .unwrap();
650 let values = result
651 .frame
652 .column("mean_cumulative_return")
653 .unwrap()
654 .as_materialized_series()
655 .f64()
656 .unwrap()
657 .iter()
658 .collect::<Vec<_>>();
659
660 assert_eq!(values.len(), 3);
661 approx::assert_abs_diff_eq!(values[0].unwrap(), -0.090909090909, epsilon = 1e-12);
662 approx::assert_abs_diff_eq!(values[1].unwrap(), 0.0, epsilon = 1e-12);
663 approx::assert_abs_diff_eq!(values[2].unwrap(), 0.10, epsilon = 1e-12);
664 }
665}