use crate::ast::{Expr, Factor};
use crate::contrasts::Coding;
use crate::data::DataFrame;
use crate::eval::eval_column;
use crate::parser::parse;
use crate::redundancy::{pick_contrasts_for_term, UsedSubterms};
use crate::terms::Term;
use ndarray::{Array1, Array2};
use solow_core::{Error, Result};
use std::collections::BTreeSet;
#[derive(Debug, Clone)]
pub struct DesignOutput {
pub y: Option<Array1<f64>>,
pub design: Array2<f64>,
pub names: Vec<String>,
}
pub fn build(formula: &str, data: &DataFrame) -> Result<DesignOutput> {
let parsed = parse(formula)?;
let nrows = data
.nrows()
.ok_or_else(|| Error::Value("data frame has no columns".into()))?;
check_lengths(data, nrows)?;
let y = match &parsed.response {
Some(f) => Some(eval_response(f, data, nrows)?),
None => None,
};
let mut terms: Vec<Term> = Vec::new();
if parsed.rhs.intercept {
terms.push(Term::intercept());
}
terms.extend(parsed.rhs.terms.iter().cloned());
for t in &terms {
for f in &t.factors {
check_factor(f, data)?;
}
}
let ordered = order_terms(&terms, data);
let mut columns: Vec<Vec<f64>> = Vec::new();
let mut names: Vec<String> = Vec::new();
build_all(&ordered, data, nrows, &mut columns, &mut names)?;
let ncols = columns.len();
let mut flat = Vec::with_capacity(nrows * ncols);
for row in 0..nrows {
for col in &columns {
flat.push(col[row]);
}
}
let design = Array2::from_shape_vec((nrows, ncols), flat)
.map_err(|e| Error::Shape(format!("design matrix assembly: {e}")))?;
Ok(DesignOutput { y, design, names })
}
fn check_lengths(data: &DataFrame, nrows: usize) -> Result<()> {
for name in &data.order {
let len = if let Some(c) = data.numeric_col(name) {
c.len()
} else if let Some(c) = data.categorical_col(name) {
c.len()
} else {
continue;
};
if len != nrows {
return Err(Error::Shape(format!(
"column `{name}` has length {len}, expected {nrows}"
)));
}
}
Ok(())
}
fn check_factor(f: &Factor, data: &DataFrame) -> Result<()> {
match f {
Factor::Numeric(n) => {
if data.is_categorical(n) {
return Err(Error::Value(format!(
"`{n}` is categorical; reference it as C({n})"
)));
}
if !data.has(n) {
return Err(Error::Value(format!("unknown variable `{n}`")));
}
}
Factor::Categorical { var, .. } => {
if !data.has(var) {
return Err(Error::Value(format!("unknown variable `{var}`")));
}
}
Factor::Identity { expr, .. } => check_expr_vars(expr, data)?,
}
Ok(())
}
fn check_expr_vars(expr: &Expr, data: &DataFrame) -> Result<()> {
match expr {
Expr::Num(_) => Ok(()),
Expr::Var(n) => {
if data.numeric_col(n).is_none() {
Err(Error::Value(format!(
"I(...): `{n}` is not a numeric column"
)))
} else {
Ok(())
}
}
Expr::Neg(a) => check_expr_vars(a, data),
Expr::Add(a, b) | Expr::Sub(a, b) | Expr::Mul(a, b) | Expr::Div(a, b) | Expr::Pow(a, b) => {
check_expr_vars(a, data)?;
check_expr_vars(b, data)
}
}
}
fn eval_response(f: &Factor, data: &DataFrame, nrows: usize) -> Result<Array1<f64>> {
match f {
Factor::Numeric(n) => {
let col = data
.numeric_col(n)
.ok_or_else(|| Error::Value(format!("response `{n}` is not numeric")))?;
Ok(Array1::from(col.to_vec()))
}
Factor::Identity { expr, .. } => Ok(Array1::from(eval_column(expr, data, nrows)?)),
Factor::Categorical { var, .. } => Err(Error::NotImplemented(format!(
"categorical response C({var}) is not supported"
))),
}
}
fn is_numeric_factor(f: &Factor) -> bool {
matches!(f, Factor::Numeric(_) | Factor::Identity { .. })
}
fn numeric_key(t: &Term) -> BTreeSet<String> {
t.factors
.iter()
.filter(|f| is_numeric_factor(f))
.map(|f| f.key())
.collect()
}
fn order_terms(terms: &[Term], _data: &DataFrame) -> Vec<Term> {
let mut bucket_order: Vec<BTreeSet<String>> = Vec::new();
for t in terms {
let k = numeric_key(t);
if !bucket_order.contains(&k) {
bucket_order.push(k);
}
}
let empty: BTreeSet<String> = BTreeSet::new();
if let Some(pos) = bucket_order.iter().position(|k| k == &empty) {
let e = bucket_order.remove(pos);
bucket_order.insert(0, e);
}
let mut out = Vec::new();
for bucket in &bucket_order {
let mut bterms: Vec<Term> = terms
.iter()
.filter(|t| &numeric_key(t) == bucket)
.cloned()
.collect();
bterms.sort_by_key(|t| t.factors.len()); out.extend(bterms);
}
out
}
fn build_all(
ordered: &[Term],
data: &DataFrame,
nrows: usize,
columns: &mut Vec<Vec<f64>>,
names: &mut Vec<String>,
) -> Result<()> {
let mut i = 0;
while i < ordered.len() {
let key = numeric_key(&ordered[i]);
let start = i;
while i < ordered.len() && numeric_key(&ordered[i]) == key {
i += 1;
}
let mut used = UsedSubterms::default();
for term in &ordered[start..i] {
emit_term(term, data, nrows, &mut used, columns, names)?;
}
}
Ok(())
}
fn emit_term(
term: &Term,
data: &DataFrame,
nrows: usize,
used: &mut UsedSubterms,
columns: &mut Vec<Vec<f64>>,
names: &mut Vec<String>,
) -> Result<()> {
let cat_keys: Vec<String> = term
.factors
.iter()
.filter(|f| matches!(f, Factor::Categorical { .. }))
.map(|f| f.key())
.collect();
let codings = pick_contrasts_for_term(&cat_keys, used);
for coding in &codings {
let mut specs: Vec<FactorSpec> = Vec::new();
for f in &term.factors {
match f {
Factor::Categorical { var, contrast, .. } => {
let Some(&full) = coding.get(&f.key()) else {
continue;
};
let col = data.categorical_col(var).ok_or_else(|| {
Error::Value(format!("C({var}): `{var}` is not categorical"))
})?;
let levels = sorted_levels(col);
let coding = contrast.coding(&levels, full);
specs.push(FactorSpec::Categorical {
label: f.label(),
rows: col.to_vec(),
levels,
coding,
});
}
_ => specs.push(FactorSpec::Numeric {
label: f.label(),
values: numeric_values(f, data, nrows)?,
}),
}
}
expand_subterm(&specs, nrows, columns, names);
}
Ok(())
}
enum FactorSpec {
Numeric {
label: String,
values: Vec<f64>,
},
Categorical {
label: String,
rows: Vec<String>,
levels: Vec<String>,
coding: Coding,
},
}
impl FactorSpec {
fn width(&self) -> usize {
match self {
FactorSpec::Numeric { .. } => 1,
FactorSpec::Categorical { coding, .. } => coding.suffixes.len(),
}
}
fn column(&self, j: usize, nrows: usize) -> (String, Vec<f64>) {
match self {
FactorSpec::Numeric { label, values } => (label.clone(), values.clone()),
FactorSpec::Categorical {
label,
rows,
levels,
coding,
} => {
let suffix = format!("{label}{}", coding.suffixes[j]);
let mut v = vec![0.0; nrows];
for (i, val) in v.iter_mut().enumerate() {
let level_idx = levels.iter().position(|l| l == &rows[i]);
if let Some(li) = level_idx {
*val = coding.matrix[li][j];
}
}
(suffix, v)
}
}
}
}
fn expand_subterm(
specs: &[FactorSpec],
nrows: usize,
columns: &mut Vec<Vec<f64>>,
names: &mut Vec<String>,
) {
let widths: Vec<usize> = specs.iter().map(FactorSpec::width).collect();
if widths.contains(&0) {
return;
}
for combo in column_combinations(&widths) {
let mut pieces: Vec<String> = Vec::new();
let mut col = vec![1.0f64; nrows];
for (fi, &j) in combo.iter().enumerate() {
let (suffix, values) = specs[fi].column(j, nrows);
pieces.push(suffix);
for (c, x) in col.iter_mut().zip(values.iter()) {
*c *= x;
}
}
names.push(if pieces.is_empty() {
"Intercept".to_string()
} else {
pieces.join(":")
});
columns.push(col);
}
}
fn column_combinations(widths: &[usize]) -> Vec<Vec<usize>> {
if widths.is_empty() {
return vec![vec![]];
}
let total: usize = widths.iter().product();
let mut out = Vec::with_capacity(total);
let mut idx = vec![0usize; widths.len()];
for _ in 0..total {
out.push(idx.clone());
let mut k = 0;
loop {
idx[k] += 1;
if idx[k] < widths[k] {
break;
}
idx[k] = 0;
k += 1;
if k == widths.len() {
break;
}
}
}
out
}
fn sorted_levels(col: &[String]) -> Vec<String> {
let mut set: BTreeSet<String> = BTreeSet::new();
for v in col {
set.insert(v.clone());
}
set.into_iter().collect()
}
fn numeric_values(f: &Factor, data: &DataFrame, nrows: usize) -> Result<Vec<f64>> {
match f {
Factor::Numeric(n) => {
let c = data
.numeric_col(n)
.ok_or_else(|| Error::Value(format!("`{n}` is not numeric")))?;
Ok(c.to_vec())
}
Factor::Identity { expr, .. } => eval_column(expr, data, nrows),
Factor::Categorical { .. } => Err(Error::Value("numeric_values on categorical".into())),
}
}