rdrobust_core/rdplot.rs
1//! RD plot port -- `rdplot` (R rdplot.R): the binned scatter (per-bin means),
2//! the IMSE/MV bin-count selection, and the global polynomial curve coefficients.
3//!
4//! Implemented: the ES family of binselect (`es`, `esmv` (default),
5//! `espr`, `esmvpr`) with no covariates, no CI bands. QS spacing, covariate
6//! adjustment, and CI/sd are later increments. Parity: exact bin counts
7//! `J_l`/`J_r`, poly coefficients and per-bin means ~1e-8 vs R.
8
9use nalgebra::{DMatrix, DVector};
10
11use crate::kernel::eff_weight;
12use crate::linalg::{qr_xx_inv, vander};
13use crate::Kernel;
14
15/// Numeric outputs behind an RD plot: selected bin counts, global polynomial curve
16/// coefficients, and per-bin scatter means for each side.
17///
18/// Ports the subset of the R `rdplot` return object the numeric core needs (R
19/// rdplot.R:557-607): the selected counts `J`, the curve coefficients `coef`, and
20/// the per-bin x/y means that place each scatter dot. Plotting, CI bands, per-bin
21/// N/sd/SE, bin midpoints, and the QS/covariate paths are not part of this slice.
22/// Bin counts match R exactly; coefficients and per-bin means match R to ~1e-8.
23#[derive(Debug, Clone)]
24pub struct RdPlot {
25 /// Selected number of bins on the left side (`x < c`), post scale/nbins, clamped
26 /// to at least 1. R `J[1]` (rdplot.R:449-466).
27 pub j_l: usize,
28 /// Selected number of bins on the right side (`x >= c`), post scale/nbins, clamped
29 /// to at least 1. R `J[2]`.
30 pub j_r: usize,
31 /// Left-side global polynomial coefficients, length `p + 1` (orders 0..=p in `x - c`).
32 /// R `gamma_p1_l` / `coef[,"Left"]` (rdplot.R:206-225).
33 pub coef_l: Vec<f64>,
34 /// Right-side global polynomial coefficients, length `p + 1`. R `gamma_p1_r` /
35 /// `coef[,"Right"]`.
36 pub coef_r: Vec<f64>,
37 /// Sorted unique non-empty left bin ids (negative: `-j_l` leftmost, `-1` just below
38 /// `c`). R `sort(unique(bin_x_l))` (rdplot.R:484).
39 pub bin_l: Vec<i64>,
40 /// Sorted unique non-empty right bin ids (`1..=j_r`, ascending in x). R
41 /// `sort(unique(bin_x_r))`.
42 pub bin_r: Vec<i64>,
43 /// Per-bin mean of `x` on the left, ordered by sorted bin id (the scatter dot's
44 /// x-coordinate). One entry per non-empty bin. R `tapply(x_l, bin_x_l, mean)`
45 /// (rdplot.R:488-490).
46 pub mean_x_l: Vec<f64>,
47 /// Per-bin mean of `y` on the left, ordered by sorted bin id (the scatter dot's
48 /// y-coordinate). R `tapply(y_l, bin_x_l, mean)`.
49 pub mean_y_l: Vec<f64>,
50 /// Per-bin mean of `x` on the right, ordered by sorted bin id. R
51 /// `tapply(x_r, bin_x_r, mean)`.
52 pub mean_x_r: Vec<f64>,
53 /// Per-bin mean of `y` on the right, ordered by sorted bin id. R
54 /// `tapply(y_r, bin_x_r, mean)`.
55 pub mean_y_r: Vec<f64>,
56}
57
58/// Row-scale a matrix by a per-row vector (`diag(w) * m`).
59fn row_scale(m: &DMatrix<f64>, w: &[f64]) -> DMatrix<f64> {
60 let mut out = m.clone();
61 for i in 0..m.nrows() {
62 for j in 0..m.ncols() {
63 out[(i, j)] *= w[i];
64 }
65 }
66 out
67}
68
69/// Sample variance (ddof=1), R `var`.
70fn var_y(y: &[f64]) -> f64 {
71 let n = y.len() as f64;
72 let mean = y.iter().sum::<f64>() / n;
73 y.iter().map(|&v| (v - mean) * (v - mean)).sum::<f64>() / (n - 1.0)
74}
75
76/// Global weighted local-polynomial fit of order `p` over a side (kernel weights
77/// at bandwidth `h`, times user weights `wt`; points outside the window get weight
78/// 0). Returns the (p+1) coefficients (R rdplot.R:206-225,212-214).
79fn poly_fit(
80 x: &[f64],
81 y: &[f64],
82 c: f64,
83 p: usize,
84 h: f64,
85 kernel: Kernel,
86 wt: Option<&[f64]>,
87) -> Vec<f64> {
88 let w = eff_weight(x, c, h, kernel, wt);
89 let u: Vec<f64> = x.iter().map(|&xi| xi - c).collect();
90 let r_p = vander(&u, p);
91 let wsqrt: Vec<f64> = w.iter().map(|v| v.sqrt()).collect();
92 let inv_g = qr_xx_inv(&row_scale(&r_p, &wsqrt));
93 let rw = row_scale(&r_p, &w);
94 let yv = DVector::from_column_slice(y);
95 let gamma = &inv_g * (rw.transpose() * &yv);
96 gamma.iter().copied().collect()
97}
98
99/// The four ES bin-count quantities for one side (rdplot.R:276-388, ES half).
100struct EsPilot {
101 hat_dw: f64, // J_es_hat_dw (IMSE, spacings sigma2)
102 chk_dw: f64, // J_es_chk_dw (IMSE, poly-reg sigma2)
103 hat_mv: f64, // J_es_hat_mv (mimicking-variance, spacings)
104 chk_mv: f64, // J_es_chk_mv (mimicking-variance, poly-reg)
105}
106
107/// Vandermonde derivative design: col j = j * x^(j-1) (col 0 = 0), n x (k+1).
108fn dvander(x: &[f64], k: usize) -> DMatrix<f64> {
109 let n = x.len();
110 let mut d = DMatrix::zeros(n, k + 1);
111 for i in 0..n {
112 for j in 1..=k {
113 d[(i, j)] = j as f64 * x[i].powi((j - 1) as i32);
114 }
115 }
116 d
117}
118
119/// Pilot polynomial order for the bin selector (R rdplot.R:276-298): `k = 4`,
120/// dropping to 3 and then 2 only when `qrXXinv` of the raw-`x` Vandermonde Gram
121/// fails on EITHER side. R wraps `qrXXinv` in `try`, and `qrXXinv` itself falls
122/// back from Cholesky to the pseudoinverse, so the only failure left is a
123/// non-finite Gram (`svd` on Inf/NaN); an ill-conditioned but finite Gram stays
124/// at `k = 4` with `ginv`. The order is shared by both sides, as in R.
125fn pilot_order(xl: &[f64], xr: &[f64]) -> usize {
126 let finite_gram = |x: &[f64], k: usize| {
127 let r = vander(x, k);
128 (r.transpose() * &r).iter().all(|v| v.is_finite())
129 };
130 let mut k = 4usize;
131 while k > 2 && !(finite_gram(xl, k) && finite_gram(xr, k)) {
132 k -= 1;
133 }
134 k
135}
136
137/// ES pilot for one side (R rdplot.R:300-388). `x`/`y` are the side's points,
138/// `range = |c - x_endpoint|` (support half-range), `n` is the TOTAL sample size,
139/// `k` the shared pilot order from [`pilot_order`]. The pilot fit is UNWEIGHTED on
140/// RAW `x` (not `x - c`), via `qrXXinv` (Cholesky, else pseudoinverse).
141fn es_pilot(x: &[f64], y: &[f64], range: f64, n: usize, vy: f64, k: usize) -> EsPilot {
142 let rk = vander(x, k);
143 let inv_gk = qr_xx_inv(&rk);
144 let yv = DVector::from_column_slice(y);
145 let y2: Vec<f64> = y.iter().map(|&v| v * v).collect();
146 let y2v = DVector::from_column_slice(&y2);
147 let gamma_k1 = &inv_gk * (rk.transpose() * &yv);
148 let gamma_k2 = &inv_gk * (rk.transpose() * &y2v);
149
150 // mu's at raw points
151 let drk = dvander(x, k);
152 let mu1_hat = &drk * &gamma_k1; // fitted first derivative
153
154 // sort (x, y) ascending by x for the spacings estimators
155 let mut order: Vec<usize> = (0..x.len()).collect();
156 order.sort_by(|&a, &b| x[a].total_cmp(&x[b]));
157 let xs: Vec<f64> = order.iter().map(|&i| x[i]).collect();
158 let ys: Vec<f64> = order.iter().map(|&i| y[i]).collect();
159 let m = xs.len();
160 let dxi: Vec<f64> = (1..m).map(|i| xs[i] - xs[i - 1]).collect();
161 let dyi: Vec<f64> = (1..m).map(|i| ys[i] - ys[i - 1]).collect();
162 let x_bar: Vec<f64> = (1..m).map(|i| 0.5 * (xs[i] + xs[i - 1])).collect();
163
164 // sigma2 at midpoints (poly-reg estimator), negatives -> var_y
165 let rk_bar = vander(&x_bar, k);
166 let mu0_i = &rk_bar * &gamma_k1;
167 let mu2_i = &rk_bar * &gamma_k2;
168 let sigma2_bar: Vec<f64> = (0..x_bar.len())
169 .map(|i| {
170 let s = mu2_i[i] - mu0_i[i] * mu0_i[i];
171 if s < 0.0 {
172 vy
173 } else {
174 s
175 }
176 })
177 .collect();
178
179 let nf = n as f64;
180 let sum_mu1_sq: f64 = mu1_hat.iter().map(|v| v * v).sum();
181 let b_es = (range * range / (12.0 * nf)) * sum_mu1_sq;
182 let v_hat = (0.5 / range)
183 * dxi
184 .iter()
185 .zip(&dyi)
186 .map(|(&dx, &dy)| dx * dy * dy)
187 .sum::<f64>();
188 let v_chk = (1.0 / range)
189 * dxi
190 .iter()
191 .zip(&sigma2_bar)
192 .map(|(&dx, &s)| dx * s)
193 .sum::<f64>();
194
195 let j_fun = |b: f64, v: f64| (((2.0 * b) / v) * nf).powf(1.0 / 3.0).ceil();
196 let nlogn = nf / (nf.ln() * nf.ln());
197 EsPilot {
198 hat_dw: j_fun(b_es, v_hat),
199 chk_dw: j_fun(b_es, v_chk),
200 hat_mv: ((vy / v_hat) * nlogn).ceil(),
201 chk_mv: ((vy / v_chk) * nlogn).ceil(),
202 }
203}
204
205/// R `findInterval(v, knots, rightmost.closed=TRUE)`: count of knots <= v, with the
206/// top knot folded into the last interval. knots ascending, length J+1.
207fn find_interval(v: f64, knots: &[f64]) -> usize {
208 let j = knots.len() - 1;
209 if v < knots[0] {
210 return 0;
211 }
212 if v >= knots[j] {
213 return j;
214 }
215 knots.iter().filter(|&&k| k <= v).count()
216}
217
218/// Per-bin means grouped by bin id (ascending), dropping empty bins (R `tapply`).
219///
220/// Single forward pass over `bins` in increasing index order, accumulating
221/// `(sum_x, sum_y, count)` per id in a `BTreeMap` (keys iterate ascending, matching
222/// the sorted-unique-id order). Accumulating in index order keeps each bin's
223/// floating-point sum bit-identical to the per-id-scan form.
224fn bin_means(x: &[f64], y: &[f64], bins: &[i64]) -> (Vec<i64>, Vec<f64>, Vec<f64>) {
225 use std::collections::BTreeMap;
226 let mut acc: BTreeMap<i64, (f64, f64, usize)> = BTreeMap::new();
227 for k in 0..bins.len() {
228 let e = acc.entry(bins[k]).or_insert((0.0, 0.0, 0));
229 e.0 += x[k];
230 e.1 += y[k];
231 e.2 += 1;
232 }
233 let mut ids = Vec::with_capacity(acc.len());
234 let mut mx = Vec::with_capacity(acc.len());
235 let mut my = Vec::with_capacity(acc.len());
236 for (&id, &(sx, sy, cnt)) in &acc {
237 ids.push(id);
238 mx.push(sx / cnt as f64);
239 my.push(sy / cnt as f64);
240 }
241 (ids, mx, my)
242}
243
244/// Computes the data behind an RD plot for the evenly-spaced (ES) binselect family:
245/// the global polynomial curve coefficients, the IMSE/MV bin counts, and the per-bin
246/// scatter means. Ports R `rdplot` (rdplot.R), ES half only, no covariates and no CI.
247///
248/// Splits the sample at `c` (`x < c` left, `x >= c` right), fits one weighted
249/// order-`p` polynomial per side over the full side range, selects `J_l`/`J_r` via the
250/// order-4 ES pilot regressions, lays evenly-spaced breakpoints, assigns each point
251/// with R's `findInterval` convention, and returns per-bin x/y means (empty bins
252/// dropped). Bin counts are exact vs R; coefficients and means match to ~1e-8.
253///
254/// # Arguments
255/// - `y`: outcome values.
256/// - `x`: running variable, parallel to `y`, need not be sorted.
257/// - `c`: cutoff; the point `x == c` falls on the right side.
258/// - `p`: global polynomial order for the curve (R default 4).
259/// - `kernel`: kernel for the polynomial-fit weights (uniform/triangular/Epanechnikov).
260/// - `binselect`: ES variant string, one of `es`, `espr`, `esmv` (also matched by `""`),
261/// or `esmvpr`. Any other value, including the QS variants, triggers `unimplemented!`.
262/// - `scale_l`, `scale_r`: multipliers on the selected per-side bin count (R default 1).
263/// The scaled count is rounded up to an integer; R keeps a fractional count and
264/// produces fractional bin ids, so non-integer products diverge from R there.
265/// - `nbins_l`, `nbins_r`: optional manual bin counts that override binselect per side.
266/// A side whose outcome has zero variance always gets one bin, as in R.
267/// - `wt`: optional unit weights that multiply the kernel weights in the curve fit only;
268/// they do not enter the pilot regressions or bin construction.
269///
270/// # Panics
271/// Panics via `unimplemented!` when `binselect` is not one of the four supported ES
272/// strings (the QS family is a later increment), and when a selector diverges to an
273/// infinite bin count (zero spacings variance with non-zero bias on a side, where R
274/// errors inside `seq`).
275#[allow(clippy::too_many_arguments)]
276pub fn rdplot(
277 y: &[f64],
278 x: &[f64],
279 c: f64,
280 p: usize,
281 kernel: Kernel,
282 binselect: &str,
283 scale_l: f64,
284 scale_r: f64,
285 nbins_l: Option<usize>,
286 nbins_r: Option<usize>,
287 wt: Option<&[f64]>,
288) -> RdPlot {
289 let x_min = x.iter().cloned().fold(f64::INFINITY, f64::min);
290 let x_max = x.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
291 let range_l = c - x_min;
292 let range_r = x_max - c;
293
294 // side split: x<c left, x>=c right (carry user weights for the poly fit only)
295 let (mut xl, mut yl, mut wl, mut xr, mut yr, mut wr) = (
296 Vec::new(),
297 Vec::new(),
298 Vec::new(),
299 Vec::new(),
300 Vec::new(),
301 Vec::new(),
302 );
303 for i in 0..x.len() {
304 if x[i] < c {
305 xl.push(x[i]);
306 yl.push(y[i]);
307 if let Some(ww) = wt {
308 wl.push(ww[i]);
309 }
310 } else {
311 xr.push(x[i]);
312 yr.push(y[i]);
313 if let Some(ww) = wt {
314 wr.push(ww[i]);
315 }
316 }
317 }
318 let wl_opt = wt.map(|_| wl.as_slice());
319 let wr_opt = wt.map(|_| wr.as_slice());
320 let n = x.len();
321
322 // global poly curve coefficients (h = full side range, default; weighted)
323 let coef_l = poly_fit(&xl, &yl, c, p, range_l, kernel, wl_opt);
324 let coef_r = poly_fit(&xr, &yr, c, p, range_r, kernel, wr_opt);
325
326 // ES pilot + binselect dispatch
327 let vy_l = var_y(&yl);
328 let vy_r = var_y(&yr);
329 let k = pilot_order(&xl, &xr);
330 let pl = es_pilot(&xl, &yl, range_l, n, vy_l, k);
331 let pr = es_pilot(&xr, &yr, range_r, n, vy_r, k);
332 let pick = |pe: &EsPilot| -> f64 {
333 match binselect {
334 "es" => pe.hat_dw,
335 "espr" => pe.chk_dw,
336 "esmv" | "" => pe.hat_mv,
337 "esmvpr" => pe.chk_mv,
338 other => unimplemented!("binselect {other} not yet ported (ES family only)"),
339 }
340 };
341 // J per side: scale * selected count (ceiled to an integer; R keeps a
342 // fractional J_star and lays fractional bin ids), or the manual nbins. A side
343 // with no outcome variation gets a single bin (R rdplot.R:458-466), which
344 // also overrides nbins; it pre-empts the `es` selector's 0/0 or x/0 there.
345 let bins = |nbins: Option<usize>, scale: f64, j: f64, vy: f64| -> usize {
346 if vy == 0.0 {
347 return 1;
348 }
349 let j = nbins.map_or(scale * j, |nb| nb as f64);
350 assert!(
351 j.is_finite(),
352 "rdplot: bin selector diverged (zero spacings variance with non-zero bias on one side)"
353 );
354 (j.ceil() as usize).max(1)
355 };
356 let j_l = bins(nbins_l, scale_l, pick(&pl), vy_l);
357 let j_r = bins(nbins_r, scale_r, pick(&pr), vy_r);
358
359 // ES breakpoints: x_min + i*jump (J+1 knots), last = endpoint
360 let jump_l = range_l / j_l as f64;
361 let jump_r = range_r / j_r as f64;
362 let knots_l: Vec<f64> = (0..=j_l).map(|i| x_min + i as f64 * jump_l).collect();
363 let knots_r: Vec<f64> = (0..=j_r).map(|i| c + i as f64 * jump_r).collect();
364
365 // bin assignment (R findInterval; left ids negative)
366 let bins_l: Vec<i64> = xl
367 .iter()
368 .map(|&v| find_interval(v, &knots_l) as i64 - j_l as i64 - 1)
369 .collect();
370 let bins_r: Vec<i64> = xr
371 .iter()
372 .map(|&v| find_interval(v, &knots_r) as i64)
373 .collect();
374
375 let (bin_l, mean_x_l, mean_y_l) = bin_means(&xl, &yl, &bins_l);
376 let (bin_r, mean_x_r, mean_y_r) = bin_means(&xr, &yr, &bins_r);
377
378 RdPlot {
379 j_l,
380 j_r,
381 coef_l,
382 coef_r,
383 bin_l,
384 bin_r,
385 mean_x_l,
386 mean_y_l,
387 mean_x_r,
388 mean_y_r,
389 }
390}
391
392#[cfg(test)]
393mod tests {
394 use super::*;
395
396 fn dgp(n: usize) -> (Vec<f64>, Vec<f64>) {
397 let x: Vec<f64> = (0..n)
398 .map(|i| -1.0 + 2.0 * (i as f64 + 0.5) / n as f64)
399 .collect();
400 let y: Vec<f64> = x
401 .iter()
402 .enumerate()
403 .map(|(i, &xi)| 0.5 * xi + if xi >= 0.0 { 2.0 } else { 0.0 } + 0.3 * (i as f64).sin())
404 .collect();
405 (x, y)
406 }
407
408 #[test]
409 fn pilot_order_stays_at_four_when_gram_is_finite_but_ill_conditioned() {
410 // Offset running variable: the k=4 Gram is far from positive definite in
411 // floating point (Cholesky fails), but R keeps k=4 via ginv.
412 let (x, _) = dgp(400);
413 let xo: Vec<f64> = x.iter().map(|v| v + 1.0e5).collect();
414 let (xl, xr): (Vec<f64>, Vec<f64>) = xo.iter().partition(|&&v| v < 1.0e5);
415 assert_eq!(pilot_order(&xl, &xr), 4);
416 // Non-finite Gram (x^8 overflows) drops the order on both sides.
417 let huge: Vec<f64> = x.iter().map(|v| v * 1.0e40).collect();
418 let (hl, hr): (Vec<f64>, Vec<f64>) = huge.iter().partition(|&&v| v < 0.0);
419 assert!(pilot_order(&hl, &hr) < 4);
420 }
421
422 #[test]
423 fn constant_outcome_side_gets_one_bin() {
424 let (x, _) = dgp(300);
425 let y: Vec<f64> = x
426 .iter()
427 .map(|&xi| if xi >= 0.0 { 1.0 + xi } else { 0.0 })
428 .collect();
429 for sel in ["es", "espr", "esmv", "esmvpr"] {
430 let out = rdplot(
431 &y,
432 &x,
433 0.0,
434 4,
435 Kernel::Uniform,
436 sel,
437 1.0,
438 1.0,
439 None,
440 None,
441 None,
442 );
443 assert_eq!(out.j_l, 1, "{sel}");
444 assert!(out.j_r >= 1);
445 assert_eq!(out.mean_y_l, vec![0.0]);
446 }
447 // the one-bin rule also overrides a manual nbins (R rdplot.R:452-461)
448 let out = rdplot(
449 &y,
450 &x,
451 0.0,
452 4,
453 Kernel::Uniform,
454 "esmv",
455 1.0,
456 1.0,
457 Some(7),
458 Some(7),
459 None,
460 );
461 assert_eq!((out.j_l, out.j_r), (1, 7));
462 }
463}