Skip to main content

rdrobust_core/
rdplot.rs

1//! RD plot port -- `rdplot` (R rdplot.R): the binned scatter (per-bin means),
2//! the IMSE/MV bin-count selection, and the global polynomial curve coefficients.
3//!
4//! Implemented: the ES family of binselect (`es`, `esmv` (default),
5//! `espr`, `esmvpr`) with no covariates, no CI bands. QS spacing, covariate
6//! adjustment, and CI/sd are later increments. Parity: exact bin counts
7//! `J_l`/`J_r`, poly coefficients and per-bin means ~1e-8 vs R.
8
9use nalgebra::{DMatrix, DVector};
10
11use crate::kernel::eff_weight;
12use crate::linalg::{qr_xx_inv, vander};
13use crate::Kernel;
14
15/// Numeric outputs behind an RD plot: selected bin counts, global polynomial curve
16/// coefficients, and per-bin scatter means for each side.
17///
18/// Ports the subset of the R `rdplot` return object the numeric core needs (R
19/// rdplot.R:557-607): the selected counts `J`, the curve coefficients `coef`, and
20/// the per-bin x/y means that place each scatter dot. Plotting, CI bands, per-bin
21/// N/sd/SE, bin midpoints, and the QS/covariate paths are not part of this slice.
22/// Bin counts match R exactly; coefficients and per-bin means match R to ~1e-8.
23#[derive(Debug, Clone)]
24pub struct RdPlot {
25    /// Selected number of bins on the left side (`x < c`), post scale/nbins, clamped
26    /// to at least 1. R `J[1]` (rdplot.R:449-466).
27    pub j_l: usize,
28    /// Selected number of bins on the right side (`x >= c`), post scale/nbins, clamped
29    /// to at least 1. R `J[2]`.
30    pub j_r: usize,
31    /// Left-side global polynomial coefficients, length `p + 1` (orders 0..=p in `x - c`).
32    /// R `gamma_p1_l` / `coef[,"Left"]` (rdplot.R:206-225).
33    pub coef_l: Vec<f64>,
34    /// Right-side global polynomial coefficients, length `p + 1`. R `gamma_p1_r` /
35    /// `coef[,"Right"]`.
36    pub coef_r: Vec<f64>,
37    /// Sorted unique non-empty left bin ids (negative: `-j_l` leftmost, `-1` just below
38    /// `c`). R `sort(unique(bin_x_l))` (rdplot.R:484).
39    pub bin_l: Vec<i64>,
40    /// Sorted unique non-empty right bin ids (`1..=j_r`, ascending in x). R
41    /// `sort(unique(bin_x_r))`.
42    pub bin_r: Vec<i64>,
43    /// Per-bin mean of `x` on the left, ordered by sorted bin id (the scatter dot's
44    /// x-coordinate). One entry per non-empty bin. R `tapply(x_l, bin_x_l, mean)`
45    /// (rdplot.R:488-490).
46    pub mean_x_l: Vec<f64>,
47    /// Per-bin mean of `y` on the left, ordered by sorted bin id (the scatter dot's
48    /// y-coordinate). R `tapply(y_l, bin_x_l, mean)`.
49    pub mean_y_l: Vec<f64>,
50    /// Per-bin mean of `x` on the right, ordered by sorted bin id. R
51    /// `tapply(x_r, bin_x_r, mean)`.
52    pub mean_x_r: Vec<f64>,
53    /// Per-bin mean of `y` on the right, ordered by sorted bin id. R
54    /// `tapply(y_r, bin_x_r, mean)`.
55    pub mean_y_r: Vec<f64>,
56}
57
58/// Row-scale a matrix by a per-row vector (`diag(w) * m`).
59fn row_scale(m: &DMatrix<f64>, w: &[f64]) -> DMatrix<f64> {
60    let mut out = m.clone();
61    for i in 0..m.nrows() {
62        for j in 0..m.ncols() {
63            out[(i, j)] *= w[i];
64        }
65    }
66    out
67}
68
69/// Sample variance (ddof=1), R `var`.
70fn var_y(y: &[f64]) -> f64 {
71    let n = y.len() as f64;
72    let mean = y.iter().sum::<f64>() / n;
73    y.iter().map(|&v| (v - mean) * (v - mean)).sum::<f64>() / (n - 1.0)
74}
75
76/// Global weighted local-polynomial fit of order `p` over a side (kernel weights
77/// at bandwidth `h`, times user weights `wt`; points outside the window get weight
78/// 0). Returns the (p+1) coefficients (R rdplot.R:206-225,212-214).
79fn poly_fit(
80    x: &[f64],
81    y: &[f64],
82    c: f64,
83    p: usize,
84    h: f64,
85    kernel: Kernel,
86    wt: Option<&[f64]>,
87) -> Vec<f64> {
88    let w = eff_weight(x, c, h, kernel, wt);
89    let u: Vec<f64> = x.iter().map(|&xi| xi - c).collect();
90    let r_p = vander(&u, p);
91    let wsqrt: Vec<f64> = w.iter().map(|v| v.sqrt()).collect();
92    let inv_g = qr_xx_inv(&row_scale(&r_p, &wsqrt));
93    let rw = row_scale(&r_p, &w);
94    let yv = DVector::from_column_slice(y);
95    let gamma = &inv_g * (rw.transpose() * &yv);
96    gamma.iter().copied().collect()
97}
98
99/// The four ES bin-count quantities for one side (rdplot.R:276-388, ES half).
100struct EsPilot {
101    hat_dw: f64, // J_es_hat_dw (IMSE, spacings sigma2)
102    chk_dw: f64, // J_es_chk_dw (IMSE, poly-reg sigma2)
103    hat_mv: f64, // J_es_hat_mv (mimicking-variance, spacings)
104    chk_mv: f64, // J_es_chk_mv (mimicking-variance, poly-reg)
105}
106
107/// Vandermonde derivative design: col j = j * x^(j-1) (col 0 = 0), n x (k+1).
108fn dvander(x: &[f64], k: usize) -> DMatrix<f64> {
109    let n = x.len();
110    let mut d = DMatrix::zeros(n, k + 1);
111    for i in 0..n {
112        for j in 1..=k {
113            d[(i, j)] = j as f64 * x[i].powi((j - 1) as i32);
114        }
115    }
116    d
117}
118
119/// Pilot polynomial order for the bin selector (R rdplot.R:276-298): `k = 4`,
120/// dropping to 3 and then 2 only when `qrXXinv` of the raw-`x` Vandermonde Gram
121/// fails on EITHER side. R wraps `qrXXinv` in `try`, and `qrXXinv` itself falls
122/// back from Cholesky to the pseudoinverse, so the only failure left is a
123/// non-finite Gram (`svd` on Inf/NaN); an ill-conditioned but finite Gram stays
124/// at `k = 4` with `ginv`. The order is shared by both sides, as in R.
125fn pilot_order(xl: &[f64], xr: &[f64]) -> usize {
126    let finite_gram = |x: &[f64], k: usize| {
127        let r = vander(x, k);
128        (r.transpose() * &r).iter().all(|v| v.is_finite())
129    };
130    let mut k = 4usize;
131    while k > 2 && !(finite_gram(xl, k) && finite_gram(xr, k)) {
132        k -= 1;
133    }
134    k
135}
136
137/// ES pilot for one side (R rdplot.R:300-388). `x`/`y` are the side's points,
138/// `range = |c - x_endpoint|` (support half-range), `n` is the TOTAL sample size,
139/// `k` the shared pilot order from [`pilot_order`]. The pilot fit is UNWEIGHTED on
140/// RAW `x` (not `x - c`), via `qrXXinv` (Cholesky, else pseudoinverse).
141fn es_pilot(x: &[f64], y: &[f64], range: f64, n: usize, vy: f64, k: usize) -> EsPilot {
142    let rk = vander(x, k);
143    let inv_gk = qr_xx_inv(&rk);
144    let yv = DVector::from_column_slice(y);
145    let y2: Vec<f64> = y.iter().map(|&v| v * v).collect();
146    let y2v = DVector::from_column_slice(&y2);
147    let gamma_k1 = &inv_gk * (rk.transpose() * &yv);
148    let gamma_k2 = &inv_gk * (rk.transpose() * &y2v);
149
150    // mu's at raw points
151    let drk = dvander(x, k);
152    let mu1_hat = &drk * &gamma_k1; // fitted first derivative
153
154    // sort (x, y) ascending by x for the spacings estimators
155    let mut order: Vec<usize> = (0..x.len()).collect();
156    order.sort_by(|&a, &b| x[a].total_cmp(&x[b]));
157    let xs: Vec<f64> = order.iter().map(|&i| x[i]).collect();
158    let ys: Vec<f64> = order.iter().map(|&i| y[i]).collect();
159    let m = xs.len();
160    let dxi: Vec<f64> = (1..m).map(|i| xs[i] - xs[i - 1]).collect();
161    let dyi: Vec<f64> = (1..m).map(|i| ys[i] - ys[i - 1]).collect();
162    let x_bar: Vec<f64> = (1..m).map(|i| 0.5 * (xs[i] + xs[i - 1])).collect();
163
164    // sigma2 at midpoints (poly-reg estimator), negatives -> var_y
165    let rk_bar = vander(&x_bar, k);
166    let mu0_i = &rk_bar * &gamma_k1;
167    let mu2_i = &rk_bar * &gamma_k2;
168    let sigma2_bar: Vec<f64> = (0..x_bar.len())
169        .map(|i| {
170            let s = mu2_i[i] - mu0_i[i] * mu0_i[i];
171            if s < 0.0 {
172                vy
173            } else {
174                s
175            }
176        })
177        .collect();
178
179    let nf = n as f64;
180    let sum_mu1_sq: f64 = mu1_hat.iter().map(|v| v * v).sum();
181    let b_es = (range * range / (12.0 * nf)) * sum_mu1_sq;
182    let v_hat = (0.5 / range)
183        * dxi
184            .iter()
185            .zip(&dyi)
186            .map(|(&dx, &dy)| dx * dy * dy)
187            .sum::<f64>();
188    let v_chk = (1.0 / range)
189        * dxi
190            .iter()
191            .zip(&sigma2_bar)
192            .map(|(&dx, &s)| dx * s)
193            .sum::<f64>();
194
195    let j_fun = |b: f64, v: f64| (((2.0 * b) / v) * nf).powf(1.0 / 3.0).ceil();
196    let nlogn = nf / (nf.ln() * nf.ln());
197    EsPilot {
198        hat_dw: j_fun(b_es, v_hat),
199        chk_dw: j_fun(b_es, v_chk),
200        hat_mv: ((vy / v_hat) * nlogn).ceil(),
201        chk_mv: ((vy / v_chk) * nlogn).ceil(),
202    }
203}
204
205/// R `findInterval(v, knots, rightmost.closed=TRUE)`: count of knots <= v, with the
206/// top knot folded into the last interval. knots ascending, length J+1.
207fn find_interval(v: f64, knots: &[f64]) -> usize {
208    let j = knots.len() - 1;
209    if v < knots[0] {
210        return 0;
211    }
212    if v >= knots[j] {
213        return j;
214    }
215    knots.iter().filter(|&&k| k <= v).count()
216}
217
218/// Per-bin means grouped by bin id (ascending), dropping empty bins (R `tapply`).
219///
220/// Single forward pass over `bins` in increasing index order, accumulating
221/// `(sum_x, sum_y, count)` per id in a `BTreeMap` (keys iterate ascending, matching
222/// the sorted-unique-id order). Accumulating in index order keeps each bin's
223/// floating-point sum bit-identical to the per-id-scan form.
224fn bin_means(x: &[f64], y: &[f64], bins: &[i64]) -> (Vec<i64>, Vec<f64>, Vec<f64>) {
225    use std::collections::BTreeMap;
226    let mut acc: BTreeMap<i64, (f64, f64, usize)> = BTreeMap::new();
227    for k in 0..bins.len() {
228        let e = acc.entry(bins[k]).or_insert((0.0, 0.0, 0));
229        e.0 += x[k];
230        e.1 += y[k];
231        e.2 += 1;
232    }
233    let mut ids = Vec::with_capacity(acc.len());
234    let mut mx = Vec::with_capacity(acc.len());
235    let mut my = Vec::with_capacity(acc.len());
236    for (&id, &(sx, sy, cnt)) in &acc {
237        ids.push(id);
238        mx.push(sx / cnt as f64);
239        my.push(sy / cnt as f64);
240    }
241    (ids, mx, my)
242}
243
244/// Computes the data behind an RD plot for the evenly-spaced (ES) binselect family:
245/// the global polynomial curve coefficients, the IMSE/MV bin counts, and the per-bin
246/// scatter means. Ports R `rdplot` (rdplot.R), ES half only, no covariates and no CI.
247///
248/// Splits the sample at `c` (`x < c` left, `x >= c` right), fits one weighted
249/// order-`p` polynomial per side over the full side range, selects `J_l`/`J_r` via the
250/// order-4 ES pilot regressions, lays evenly-spaced breakpoints, assigns each point
251/// with R's `findInterval` convention, and returns per-bin x/y means (empty bins
252/// dropped). Bin counts are exact vs R; coefficients and means match to ~1e-8.
253///
254/// # Arguments
255/// - `y`: outcome values.
256/// - `x`: running variable, parallel to `y`, need not be sorted.
257/// - `c`: cutoff; the point `x == c` falls on the right side.
258/// - `p`: global polynomial order for the curve (R default 4).
259/// - `kernel`: kernel for the polynomial-fit weights (uniform/triangular/Epanechnikov).
260/// - `binselect`: ES variant string, one of `es`, `espr`, `esmv` (also matched by `""`),
261///   or `esmvpr`. Any other value, including the QS variants, triggers `unimplemented!`.
262/// - `scale_l`, `scale_r`: multipliers on the selected per-side bin count (R default 1).
263///   The scaled count is rounded up to an integer; R keeps a fractional count and
264///   produces fractional bin ids, so non-integer products diverge from R there.
265/// - `nbins_l`, `nbins_r`: optional manual bin counts that override binselect per side.
266///   A side whose outcome has zero variance always gets one bin, as in R.
267/// - `wt`: optional unit weights that multiply the kernel weights in the curve fit only;
268///   they do not enter the pilot regressions or bin construction.
269///
270/// # Panics
271/// Panics via `unimplemented!` when `binselect` is not one of the four supported ES
272/// strings (the QS family is a later increment), and when a selector diverges to an
273/// infinite bin count (zero spacings variance with non-zero bias on a side, where R
274/// errors inside `seq`).
275#[allow(clippy::too_many_arguments)]
276pub fn rdplot(
277    y: &[f64],
278    x: &[f64],
279    c: f64,
280    p: usize,
281    kernel: Kernel,
282    binselect: &str,
283    scale_l: f64,
284    scale_r: f64,
285    nbins_l: Option<usize>,
286    nbins_r: Option<usize>,
287    wt: Option<&[f64]>,
288) -> RdPlot {
289    let x_min = x.iter().cloned().fold(f64::INFINITY, f64::min);
290    let x_max = x.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
291    let range_l = c - x_min;
292    let range_r = x_max - c;
293
294    // side split: x<c left, x>=c right (carry user weights for the poly fit only)
295    let (mut xl, mut yl, mut wl, mut xr, mut yr, mut wr) = (
296        Vec::new(),
297        Vec::new(),
298        Vec::new(),
299        Vec::new(),
300        Vec::new(),
301        Vec::new(),
302    );
303    for i in 0..x.len() {
304        if x[i] < c {
305            xl.push(x[i]);
306            yl.push(y[i]);
307            if let Some(ww) = wt {
308                wl.push(ww[i]);
309            }
310        } else {
311            xr.push(x[i]);
312            yr.push(y[i]);
313            if let Some(ww) = wt {
314                wr.push(ww[i]);
315            }
316        }
317    }
318    let wl_opt = wt.map(|_| wl.as_slice());
319    let wr_opt = wt.map(|_| wr.as_slice());
320    let n = x.len();
321
322    // global poly curve coefficients (h = full side range, default; weighted)
323    let coef_l = poly_fit(&xl, &yl, c, p, range_l, kernel, wl_opt);
324    let coef_r = poly_fit(&xr, &yr, c, p, range_r, kernel, wr_opt);
325
326    // ES pilot + binselect dispatch
327    let vy_l = var_y(&yl);
328    let vy_r = var_y(&yr);
329    let k = pilot_order(&xl, &xr);
330    let pl = es_pilot(&xl, &yl, range_l, n, vy_l, k);
331    let pr = es_pilot(&xr, &yr, range_r, n, vy_r, k);
332    let pick = |pe: &EsPilot| -> f64 {
333        match binselect {
334            "es" => pe.hat_dw,
335            "espr" => pe.chk_dw,
336            "esmv" | "" => pe.hat_mv,
337            "esmvpr" => pe.chk_mv,
338            other => unimplemented!("binselect {other} not yet ported (ES family only)"),
339        }
340    };
341    // J per side: scale * selected count (ceiled to an integer; R keeps a
342    // fractional J_star and lays fractional bin ids), or the manual nbins. A side
343    // with no outcome variation gets a single bin (R rdplot.R:458-466), which
344    // also overrides nbins; it pre-empts the `es` selector's 0/0 or x/0 there.
345    let bins = |nbins: Option<usize>, scale: f64, j: f64, vy: f64| -> usize {
346        if vy == 0.0 {
347            return 1;
348        }
349        let j = nbins.map_or(scale * j, |nb| nb as f64);
350        assert!(
351            j.is_finite(),
352            "rdplot: bin selector diverged (zero spacings variance with non-zero bias on one side)"
353        );
354        (j.ceil() as usize).max(1)
355    };
356    let j_l = bins(nbins_l, scale_l, pick(&pl), vy_l);
357    let j_r = bins(nbins_r, scale_r, pick(&pr), vy_r);
358
359    // ES breakpoints: x_min + i*jump  (J+1 knots), last = endpoint
360    let jump_l = range_l / j_l as f64;
361    let jump_r = range_r / j_r as f64;
362    let knots_l: Vec<f64> = (0..=j_l).map(|i| x_min + i as f64 * jump_l).collect();
363    let knots_r: Vec<f64> = (0..=j_r).map(|i| c + i as f64 * jump_r).collect();
364
365    // bin assignment (R findInterval; left ids negative)
366    let bins_l: Vec<i64> = xl
367        .iter()
368        .map(|&v| find_interval(v, &knots_l) as i64 - j_l as i64 - 1)
369        .collect();
370    let bins_r: Vec<i64> = xr
371        .iter()
372        .map(|&v| find_interval(v, &knots_r) as i64)
373        .collect();
374
375    let (bin_l, mean_x_l, mean_y_l) = bin_means(&xl, &yl, &bins_l);
376    let (bin_r, mean_x_r, mean_y_r) = bin_means(&xr, &yr, &bins_r);
377
378    RdPlot {
379        j_l,
380        j_r,
381        coef_l,
382        coef_r,
383        bin_l,
384        bin_r,
385        mean_x_l,
386        mean_y_l,
387        mean_x_r,
388        mean_y_r,
389    }
390}
391
392#[cfg(test)]
393mod tests {
394    use super::*;
395
396    fn dgp(n: usize) -> (Vec<f64>, Vec<f64>) {
397        let x: Vec<f64> = (0..n)
398            .map(|i| -1.0 + 2.0 * (i as f64 + 0.5) / n as f64)
399            .collect();
400        let y: Vec<f64> = x
401            .iter()
402            .enumerate()
403            .map(|(i, &xi)| 0.5 * xi + if xi >= 0.0 { 2.0 } else { 0.0 } + 0.3 * (i as f64).sin())
404            .collect();
405        (x, y)
406    }
407
408    #[test]
409    fn pilot_order_stays_at_four_when_gram_is_finite_but_ill_conditioned() {
410        // Offset running variable: the k=4 Gram is far from positive definite in
411        // floating point (Cholesky fails), but R keeps k=4 via ginv.
412        let (x, _) = dgp(400);
413        let xo: Vec<f64> = x.iter().map(|v| v + 1.0e5).collect();
414        let (xl, xr): (Vec<f64>, Vec<f64>) = xo.iter().partition(|&&v| v < 1.0e5);
415        assert_eq!(pilot_order(&xl, &xr), 4);
416        // Non-finite Gram (x^8 overflows) drops the order on both sides.
417        let huge: Vec<f64> = x.iter().map(|v| v * 1.0e40).collect();
418        let (hl, hr): (Vec<f64>, Vec<f64>) = huge.iter().partition(|&&v| v < 0.0);
419        assert!(pilot_order(&hl, &hr) < 4);
420    }
421
422    #[test]
423    fn constant_outcome_side_gets_one_bin() {
424        let (x, _) = dgp(300);
425        let y: Vec<f64> = x
426            .iter()
427            .map(|&xi| if xi >= 0.0 { 1.0 + xi } else { 0.0 })
428            .collect();
429        for sel in ["es", "espr", "esmv", "esmvpr"] {
430            let out = rdplot(
431                &y,
432                &x,
433                0.0,
434                4,
435                Kernel::Uniform,
436                sel,
437                1.0,
438                1.0,
439                None,
440                None,
441                None,
442            );
443            assert_eq!(out.j_l, 1, "{sel}");
444            assert!(out.j_r >= 1);
445            assert_eq!(out.mean_y_l, vec![0.0]);
446        }
447        // the one-bin rule also overrides a manual nbins (R rdplot.R:452-461)
448        let out = rdplot(
449            &y,
450            &x,
451            0.0,
452            4,
453            Kernel::Uniform,
454            "esmv",
455            1.0,
456            1.0,
457            Some(7),
458            Some(7),
459            None,
460        );
461        assert_eq!((out.j_l, out.j_r), (1, 7));
462    }
463}