use super::{Probability, error, scalar, split_arguments};
use crate::{Context, Error, Outcome, Plugin, re};
use num_integer::Integer;
use num_rational::BigRational;
use num_traits::{One, Signed, ToPrimitive, Zero};
fn list(input: &str) -> String {
re!(r"(?i),?\s+and\s+")
.replace_all(input, ", ")
.into_owned()
}
pub(super) fn coordinate(input: &str, context: &Context) -> Result<(BigRational, bool), Error> {
let number = scalar(input, context)?;
match crate::plugins::integer_queries::ratio_value(input) {
Ok((n, d)) => Ok((BigRational::new(n, d), true)),
Err(e)
if matches!(
e.0.as_str(),
"invalid_integer_expression" | "non_rational_expression"
) =>
{
Ok((
BigRational::from_float(number).ok_or_else(|| error("numeric_overflow"))?,
false,
))
}
Err(e) => Err(e),
}
}
fn literal(value: &BigRational, authored: bool) -> Result<String, Error> {
if authored {
return Ok(format!("rational({},{})", value.numer(), value.denom()));
}
let number = value
.to_f64()
.filter(|n| n.is_finite())
.ok_or_else(|| error("numeric_overflow"))?;
if number == 0.0 && !value.is_zero() {
return Err(error("numeric_underflow"));
}
Ok(format!("real({number:e})"))
}
fn sqrt_ratio(value: BigRational, context: &Context) -> Result<f64, Error> {
let complete = crate::plugins::model_coordinates::computed_root(value, |expression| {
scalar(expression, context)
})?;
let result = complete
.to_f64()
.filter(|number| number.is_finite())
.ok_or_else(|| error("numeric_overflow"))?;
if result == 0.0 && !complete.is_zero() {
return Err(error("numeric_underflow"));
}
Ok(result)
}
fn coordinates(input: &str, context: &Context) -> Result<Vec<(BigRational, bool)>, Error> {
let input = list(input);
let pieces = split_arguments(&input)?;
if pieces.is_empty() || pieces.len() > 256 {
return Err(error("invalid_dataset"));
}
pieces.iter().map(|s| coordinate(s, context)).collect()
}
fn dataset_label(label: &str) -> bool {
matches!(
label,
"count"
| "sum"
| "minimum"
| "maximum"
| "mean"
| "median"
| "mode"
| "range"
| "population variance"
| "sample variance"
| "population standard deviation"
| "sample standard deviation"
)
}
fn dataset_goals(requested: &str) -> Result<Vec<String>, Error> {
let requested = list(requested);
let goals = split_arguments(&requested)?
.into_iter()
.map(|goal| {
let goal = goal.trim().to_ascii_lowercase();
goal.strip_prefix("the ").unwrap_or(&goal).to_owned()
})
.collect::<Vec<_>>();
if !(1..=12).contains(&goals.len()) || !goals.iter().all(|goal| dataset_label(goal)) {
return Err(error("invalid_dataset"));
}
Ok(goals)
}
fn descriptive_goals(requested: &str, population: bool) -> Result<Vec<String>, Error> {
let requested = requested.trim().to_ascii_lowercase();
let requested = requested.strip_prefix("its ").unwrap_or(&requested);
let requested = re!(r"\b(?:lowest and highest values|low and high values|smallest and largest entries|extrema)\b")
.replace_all(requested, "minimum, maximum");
let requested = list(&requested);
let goals = split_arguments(&requested)?;
let mut canonical = Vec::with_capacity(goals.len());
for goal in goals {
let goal = goal.trim().strip_prefix("the ").unwrap_or(goal.trim());
let label = match goal {
"size" | "number of values" => "count",
"total" | "their sum" => "sum",
"min" | "smallest" | "lowest" => "minimum",
"max" | "largest" | "highest" => "maximum",
"average" | "arithmetic mean" => "mean",
"mode if any" => "mode",
"variance" if population => "population variance",
label if dataset_label(label) => label,
_ => return Err(error("invalid_dataset")),
};
canonical.push(label.to_owned());
}
if !(1..=12).contains(&canonical.len()) {
return Err(error("invalid_dataset"));
}
Ok(canonical)
}
struct DatasetIntroduction<'a> {
values: &'a str,
stated_count: Option<&'a str>,
integers: bool,
}
fn dataset_introduction(input: &str) -> Option<DatasetIntroduction<'_>> {
let input = input.trim();
let folded = input.to_ascii_lowercase();
if re!(r"^(?:the )?integers? [a-z][a-z0-9_]* (?:from|in)(?:\s|$)").is_match(&folded) {
return None;
}
for prefix in [
"this full integer list, ",
"the integer dataset ",
"integer dataset ",
"the integers ",
"integers ",
"the complete dataset ",
"the full dataset ",
"the complete list ",
"complete dataset ",
"full dataset ",
"these observations, ",
"the dataset ",
"the values ",
"values ",
"dataset ",
"data ",
"scores ",
] {
if folded.starts_with(prefix) {
return Some(DatasetIntroduction {
values: input[prefix.len()..].trim(),
stated_count: None,
integers: prefix.contains("integer"),
});
}
}
if let Some(c) = re!(r"(?i)^(?:exactly )?(.+?) observations,\s*(.+)$").captures(input) {
return Some(DatasetIntroduction {
values: c.get(2)?.as_str(),
stated_count: Some(c.get(1)?.as_str()),
integers: false,
});
}
re!(r"^[+\-\d.(]")
.is_match(input)
.then_some(DatasetIntroduction {
values: input,
stated_count: None,
integers: false,
})
}
fn descriptive_request(input: &str, context: &Context) -> Option<Result<String, Error>> {
let input = input.strip_prefix("Please ").unwrap_or(input);
let input = input.strip_prefix("please ").unwrap_or(input);
let (data, goals, population) = if let Some(c) = re!(r"(?i)^(?:calculate|compute|report|give me) (?:a complete population summary|all standard requested population statistics)\s*—\s*(.+?)\s*—\s*for (.+)$").captures(input) {
(c.get(2)?.as_str(), c.get(1)?.as_str(), true)
} else if let Some(c) = re!(r"(?i)^(?:for|using|given) (.+?), (?:report|give(?: me)?|find|calculate|return|provide|state) (.+)$").captures(input) {
(c.get(1)?.as_str(), c.get(2)?.as_str(), false)
} else if let Some(c) = re!(r"(?i)^(?:use|take) (.+?) (?:to report|as the complete data and report) (.+)$").captures(input) {
(c.get(1)?.as_str(), c.get(2)?.as_str(), false)
} else if let Some(c) = re!(r"(?i)^(?:summarize every value in|summarize) (.+?) with (.+)$").captures(input) {
(c.get(1)?.as_str(), c.get(2)?.as_str(), false)
} else {
let c = re!(r"(?i)^(?:what (?:are|is)|report|find|compute|calculate) (.+)$").captures(input)?;
let body = c.get(1)?.as_str();
let folded = body.to_ascii_lowercase();
let (goals, data) = crate::plugins::syntax::split_last(&folded, &[" for ", " of "])?;
(&body[body.len() - data.len()..], &body[..goals.len()], false)
};
let introduction = dataset_introduction(data)?;
let goals = descriptive_goals(goals, population).ok()?;
if goals.len() < 2 || goals.iter().all(|goal| goal == &goals[0]) {
return None;
}
Some((|| {
if input.len() > 4096 {
return Err(error("invalid_dataset"));
}
dataset_report_with_options(
introduction.values,
&goals.join(", "),
introduction.stated_count,
introduction.integers,
context,
)
})())
}
fn report_fields(fields: &[&str]) -> bool {
(1..=5).contains(&fields.len())
|| (1..=12).contains(&fields.len())
&& fields.iter().all(|field| {
field
.split_once('|')
.is_some_and(|(label, _)| dataset_label(label))
})
}
fn report(fields: Vec<String>) -> Result<String, Error> {
let expression = format!("statistics_report({})", fields.join(";"));
if !report_fields(&fields.iter().map(String::as_str).collect::<Vec<_>>())
|| expression.len() > 4096
{
return Err(error("invalid_dataset"));
}
Ok(expression)
}
type Point = ((BigRational, bool), (BigRational, bool));
enum PmfGoal<'a> {
Mean(&'a str),
Variance(&'a str),
AtLeast(&'a str, &'a str),
}
impl PmfGoal<'_> {
fn variable(&self) -> &str {
match self {
Self::Mean(variable) | Self::Variance(variable) | Self::AtLeast(variable, _) => {
variable
}
}
}
fn label(&self) -> String {
match self {
Self::Mean(variable) => format!("E[{variable}]"),
Self::Variance(variable) => format!("Var({variable})"),
Self::AtLeast(variable, threshold) => format!("P({variable}≥{threshold})"),
}
}
}
fn pmf_goal(input: &str) -> Option<PmfGoal<'_>> {
let input = input.trim();
if input.contains([';', '|']) {
return None;
}
if let Some(c) = re!(r"(?i)^e\[([a-z][a-z0-9_]*)\]$").captures(input) {
return Some(PmfGoal::Mean(c.get(1)?.as_str()));
}
if let Some(c) = re!(r"(?i)^var\(([a-z][a-z0-9_]*)\)$").captures(input) {
return Some(PmfGoal::Variance(c.get(1)?.as_str()));
}
let c = re!(r"(?i)^p\(([a-z][a-z0-9_]*)\s*(?:≥|>=)\s*(.+)\)$").captures(input)?;
Some(PmfGoal::AtLeast(
c.get(1)?.as_str(),
c.get(2)?.as_str().trim(),
))
}
fn weighted_moments(points: &[Point]) -> Result<(BigRational, BigRational), Error> {
let bounded = crate::plugins::model_coordinates::bounded;
let mut mean = BigRational::zero();
for ((value, _), (probability, _)) in points {
mean = bounded(mean + bounded(value * probability)?)?;
}
let mut variance = BigRational::zero();
for ((value, _), (probability, _)) in points {
let difference = bounded(value - &mean)?;
variance = bounded(variance + bounded(probability * bounded(&difference * &difference)?)?)?;
}
Ok((mean, variance))
}
enum FrequencyData<'a> {
Vectors {
scores: &'a str,
frequencies: &'a str,
},
Table(&'a str),
}
struct FrequencyRequest<'a> {
data: FrequencyData<'a>,
goals: &'a str,
}
fn frequency_request(input: &str) -> Option<FrequencyRequest<'_>> {
let folded = input.to_ascii_lowercase();
let (data, goals) = crate::plugins::syntax::split_last(
&folded,
&[
"; report ",
"; give ",
"; state ",
"; return ",
". report ",
". give ",
". find ",
". calculate ",
", report ",
", give ",
", calculate ",
", determine ",
],
)?;
let data = &input[..data.len()];
let goals = &input[input.len() - goals.len()..];
if let Some(c) = re!(r"(?is)^(?:(?:take|use) (?:the )?|a discrete dataset has )(?:score-frequency pairs|value-frequency pairs|frequency table) (.+)$").captures(data) {
return Some(FrequencyRequest {
data: FrequencyData::Table(c.get(1)?.as_str()),
goals,
});
}
let c = re!(r"(?is)^(?:(?:for|using) (?:the )?|the |in a sample represented by )?(?:categories with numeric )?(?:values|scores|measurements|outcomes) (.+)$").captures(data)?;
let roles = c.get(1)?.as_str();
let c = re!(r"(?is)^(.+?) (?:with|have) (?:frequencies|counts) (.+)$")
.captures(roles)
.or_else(|| {
re!(r"(?is)^(.+?) (?:occurring|occur(?: respectively)?) (.+?) times(?: respectively)?$")
.captures(roles)
})?;
Some(FrequencyRequest {
data: FrequencyData::Vectors {
scores: c.get(1)?.as_str(),
frequencies: c.get(2)?.as_str(),
},
goals,
})
}
fn frequency_goal(input: &str) -> Option<&'static str> {
let folded = input.trim().to_ascii_lowercase();
let goal = folded
.strip_prefix("the ")
.or_else(|| folded.strip_prefix("its "))
.unwrap_or(&folded);
match goal {
"count"
| "total"
| "total count"
| "total frequency"
| "observation count"
| "observation total"
| "total observations"
| "total observation count"
| "number of observations" => return Some("total frequency"),
"weighted mean" | "weighted average" | "population mean" => return Some("weighted mean"),
"population variance" | "weighted population variance" => {
return Some("weighted population variance");
}
"weighted median (the first score whose cumulative frequency reaches half the total)" => {
return Some("weighted median");
}
_ => {}
}
let before = re!(
r"^(?:weighted )?median (?:as|by|using|defined (?:as|by)) (?:the )?first (?:(?:value|score|measurement|outcome) (?:(?:whose|with|at) )?)?(?:cumulative|running) (?:frequency|count) (?:(?:is|reaches|reaching) )?(?:at least )?half(?: (?:of )?(?:the )?(?:total|count|all observations))?$"
);
let after = re!(
r"^(?:the )?first (?:value|score|measurement|outcome) (?:(?:whose|with|at) )?(?:(?:reaches|reaching) )?(?:(?:cumulative|running) (?:frequency|count) (?:(?:is|reaches|reaching) )?(?:at least )?half(?: (?:of )?(?:the )?(?:total|count|all observations))?|at least half the cumulative count) as (?:the |its )?median$"
);
(before.is_match(goal) || after.is_match(goal)).then_some("weighted median")
}
fn paired_frequency_report(
parts: FrequencyRequest<'_>,
context: &Context,
) -> Result<String, Error> {
match parts.data {
FrequencyData::Vectors {
scores,
frequencies,
} => frequency_report(None, scores, frequencies, parts.goals, context),
FrequencyData::Table(table) => {
let table = list(table);
let records = split_arguments(&table)?;
if records.is_empty() || records.len() > 256 {
return Err(error("invalid_dataset"));
}
let mut scores = Vec::with_capacity(records.len());
let mut frequencies = Vec::with_capacity(records.len());
for record in records {
let (score, frequency) = crate::plugins::syntax::split_last(record, &[":"])
.ok_or_else(|| error("invalid_dataset"))?;
if score.is_empty()
|| frequency.is_empty()
|| crate::plugins::syntax::split_last(score, &[":"]).is_some()
{
return Err(error("invalid_dataset"));
}
scores.push(score);
frequencies.push(frequency);
}
frequency_report(
None,
&scores.join(", "),
&frequencies.join(", "),
parts.goals,
context,
)
}
}
}
fn frequency_report(
asserted_scores: Option<&str>,
scores: &str,
frequencies: &str,
goals: &str,
context: &Context,
) -> Result<String, Error> {
let asserted = asserted_scores
.map(|s| coordinates(s, context))
.transpose()?;
let scores = coordinates(scores, context)?;
let frequencies = list(frequencies);
let frequencies = split_arguments(&frequencies)?;
if asserted
.as_ref()
.is_some_and(|values| values.len() != scores.len())
|| frequencies.len() != scores.len()
{
return Err(error("invalid_dataset"));
}
let bounded = crate::plugins::model_coordinates::bounded;
let mut total = BigRational::zero();
let mut authored = true;
let mut paired = Vec::with_capacity(scores.len());
for (index, (score, frequency)) in scores.into_iter().zip(frequencies).enumerate() {
if let Some(asserted) = &asserted {
let assertion = &asserted[index];
if assertion.0 != score.0 {
return Err(error("invalid_dataset"));
}
authored &= assertion.1;
}
let (frequency, exact) = match crate::plugins::integer_queries::ratio_value(frequency) {
Ok((n, d)) => (BigRational::new(n, d), true),
Err(e)
if matches!(
e.0.as_str(),
"invalid_integer_expression" | "non_rational_expression"
) =>
{
coordinate(frequency, context)?
}
Err(e) => return Err(e),
};
let frequency = bounded(frequency)?;
if frequency.is_negative() || !frequency.is_integer() {
return Err(error("invalid_dataset"));
}
authored &= score.1 && exact;
total = bounded(total + &frequency)?;
paired.push((score, (frequency, exact)));
}
if total.is_zero() {
return Err(error("invalid_dataset"));
}
paired.sort_by(|left, right| left.0.0.cmp(&right.0.0));
let points = paired
.iter()
.map(|(score, (frequency, exact))| {
Ok((score.clone(), (bounded(frequency / &total)?, *exact)))
})
.collect::<Result<Vec<Point>, Error>>()?;
let (mean, variance) = weighted_moments(&points)?;
let mut cumulative = BigRational::zero();
let mut median = None;
for ((score, _), (frequency, _)) in &paired {
cumulative = bounded(cumulative + frequency)?;
if &cumulative * BigRational::from_integer(2.into()) >= total {
median = Some(score);
break;
}
}
let median = median.ok_or_else(|| error("invalid_dataset"))?;
let goals = list(goals);
let goals = split_arguments(&goals)?;
if !(1..=4).contains(&goals.len()) {
return Err(error("invalid_dataset"));
}
let mut seen = Vec::new();
let mut fields = Vec::with_capacity(goals.len());
for goal in goals {
let label = frequency_goal(goal).ok_or_else(|| error("invalid_dataset"))?;
let value = match label {
"total frequency" => &total,
"weighted mean" => &mean,
"weighted median" => median,
"weighted population variance" => &variance,
_ => unreachable!(),
};
if seen.contains(&label) {
return Err(error("invalid_dataset"));
}
seen.push(label);
fields.push(format!("{label}|{}", literal(value, authored)?));
}
report(fields)
}
fn pmf_report(
variable: &str,
supplied: &str,
goals: &str,
context: &Context,
) -> Result<String, Error> {
let supplied = list(supplied);
let supplied = split_arguments(&supplied)?;
if !(1..=256).contains(&supplied.len()) {
return Err(error("invalid_probability_model"));
}
let bounded = crate::plugins::model_coordinates::bounded;
let one = BigRational::from_integer(1.into());
let mut points: Vec<Point> = Vec::with_capacity(supplied.len());
let mut mass = BigRational::zero();
for declaration in supplied {
let c = re!(r"(?i)^p\(([a-z][a-z0-9_]*)\s*=\s*(.+)\)\s*=\s*(.+)$")
.captures(declaration)
.ok_or_else(|| error("invalid_probability_model"))?;
if &c[1] != variable {
return Err(error("invalid_probability_model"));
}
let (support, support_authored) = coordinate(&c[2], context)?;
let support = bounded(support)?;
let (probability, probability_authored) = coordinate(&c[3], context)?;
let probability = bounded(probability)?;
if probability.is_negative() || probability > one {
return Err(error("invalid_probability"));
}
if points.iter().any(|((previous, _), _)| previous == &support) {
return Err(error("invalid_probability_model"));
}
mass = bounded(mass + &probability)?;
points.push((
(support, support_authored),
(probability, probability_authored),
));
}
if mass != one {
return Err(error("invalid_probability_model"));
}
let (mean, variance) = weighted_moments(&points)?;
let authored = points.iter().all(|((_, x), (_, p))| *x && *p);
let goals = list(goals);
let goals = split_arguments(&goals)?;
if !(1..=5).contains(&goals.len()) {
return Err(error("invalid_probability_model"));
}
let mut labels = Vec::new();
let mut fields = Vec::with_capacity(goals.len());
for input in goals {
let goal = pmf_goal(input).ok_or_else(|| error("invalid_probability_model"))?;
let label = goal.label();
if goal.variable() != variable || labels.contains(&label) {
return Err(error("invalid_probability_model"));
}
labels.push(label.clone());
let value = match goal {
PmfGoal::Mean(_) => literal(&mean, authored)?,
PmfGoal::Variance(_) => literal(&variance, authored)?,
PmfGoal::AtLeast(_, threshold) => {
let (threshold, threshold_authored) = coordinate(threshold, context)?;
let threshold = bounded(threshold)?;
let mut probability = BigRational::zero();
for ((support, _), (mass, _)) in &points {
if support >= &threshold {
probability = bounded(probability + mass)?;
}
}
literal(&probability, authored && threshold_authored)?
}
};
fields.push(format!("{label}|{value}"));
}
report(fields)
}
fn least_squares(points: &[Point]) -> Result<(BigRational, BigRational, bool), Error> {
if !(2..=256).contains(&points.len()) {
return Err(error("invalid_dataset"));
}
let authored = points.iter().all(|((_, xa), (_, ya))| *xa && *ya);
let count = BigRational::from_integer(points.len().into());
let mean_x = crate::plugins::model_coordinates::bounded(
points
.iter()
.map(|((x, _), _)| x.clone())
.sum::<BigRational>()
/ &count,
)?;
let mean_y = crate::plugins::model_coordinates::bounded(
points
.iter()
.map(|(_, (y, _))| y.clone())
.sum::<BigRational>()
/ &count,
)?;
let mut cross = BigRational::zero();
let mut square_x = BigRational::zero();
for ((x, _), (y, _)) in points {
let dx = x - &mean_x;
let dy = y - &mean_y;
cross = crate::plugins::model_coordinates::bounded(cross + &dx * &dy)?;
square_x = crate::plugins::model_coordinates::bounded(square_x + &dx * &dx)?;
}
if square_x.is_zero() {
return Err(error("invalid_dataset"));
}
let slope = crate::plugins::model_coordinates::bounded(cross / square_x)?;
let intercept = crate::plugins::model_coordinates::bounded(mean_y - &slope * mean_x)?;
Ok((slope, intercept, authored))
}
fn paired_points(input: &str, context: &Context) -> Result<Vec<Point>, Error> {
let supplied = list(input);
split_arguments(&supplied)?
.into_iter()
.map(|point| {
let pair = point
.strip_prefix('(')
.and_then(|p| p.strip_suffix(')'))
.ok_or_else(|| error("invalid_dataset"))?;
let coordinates = split_arguments(pair)?;
let [x, y] = coordinates.as_slice() else {
return Err(error("invalid_dataset"));
};
Ok((coordinate(x, context)?, coordinate(y, context)?))
})
.collect()
}
fn residuals(
points: &[Point],
slope: &BigRational,
intercept: &BigRational,
) -> Result<(Vec<BigRational>, BigRational), Error> {
let mut residuals = Vec::with_capacity(points.len());
let mut squares = BigRational::zero();
for ((x, _), (y, _)) in points {
let fitted = crate::plugins::model_coordinates::bounded(
intercept + crate::plugins::model_coordinates::bounded(slope * x)?,
)?;
let residual = crate::plugins::model_coordinates::bounded(y - fitted)?;
squares = crate::plugins::model_coordinates::bounded(
squares + crate::plugins::model_coordinates::bounded(&residual * &residual)?,
)?;
residuals.push(residual);
}
Ok((residuals, squares))
}
fn residual_squares(
points: &[Point],
slope: &BigRational,
intercept: &BigRational,
) -> Result<BigRational, Error> {
residuals(points, slope, intercept).map(|(_, squares)| squares)
}
fn regression_report(points: &[Point], units: Option<(&str, &str)>) -> Result<String, Error> {
let (slope, intercept, authored) = least_squares(points)?;
let mean = crate::plugins::model_coordinates::bounded(
points
.iter()
.map(|(_, (y, _))| y.clone())
.sum::<BigRational>()
/ BigRational::from_integer(points.len().into()),
)?;
let mut total_squares = BigRational::zero();
for (_, (y, _)) in points {
let centered = crate::plugins::model_coordinates::bounded(y - &mean)?;
total_squares = crate::plugins::model_coordinates::bounded(
total_squares + crate::plugins::model_coordinates::bounded(¢ered * ¢ered)?,
)?;
}
if total_squares.is_zero() {
return Err(error("invalid_dataset"));
}
let r_squared = crate::plugins::model_coordinates::bounded(
BigRational::from_integer(1.into())
- residual_squares(points, &slope, &intercept)? / total_squares,
)?;
let (slope_unit, intercept_unit) = units
.map(|(x, y)| {
let grouped = |unit: &str| {
if unit.contains(['/', '*', '·', '⋅']) {
format!("({unit})")
} else {
unit.to_string()
}
};
(format!("|{}/{}", grouped(y), grouped(x)), format!("|{y}"))
})
.unwrap_or_default();
report(vec![
format!("slope|{}{slope_unit}", literal(&slope, authored)?),
format!(
"y-intercept|{}{intercept_unit}",
literal(&intercept, authored)?
),
format!("R-squared|{}", literal(&r_squared, authored)?),
])
}
fn axis_unit(
name: &str,
engine: &crate::plugins::compositions::Compositions,
) -> Result<String, Error> {
let name = name.trim().replace(" per ", "/");
let factors =
crate::plugins::compound_units::factors(&name, |leaf| engine.units.unit(leaf).is_some())
.filter(|factors| !factors.is_empty())
.ok_or_else(|| error("incompatible_units"))?;
for (leaf, _) in factors {
let unit = engine
.units
.unit(&leaf)
.ok_or_else(|| error("incompatible_units"))?;
if unit.offset != 0.0 || !unit.scale.is_finite() || unit.scale <= 0.0 {
return Err(error("invalid_dataset"));
}
}
Ok(engine
.units
.unit(&name)
.map(|unit| unit.symbol)
.unwrap_or(name))
}
fn axis_coordinates(
input: &str,
unit: &str,
engine: &crate::plugins::compositions::Compositions,
context: &Context,
) -> Result<Vec<(BigRational, bool)>, Error> {
let input = list(input);
let values = split_arguments(&input)?;
if !(2..=256).contains(&values.len()) {
return Err(error("invalid_dataset"));
}
values
.into_iter()
.map(|value| {
let typed = if engine.quantity_value(value, "1", context).is_ok() {
format!("({value}) {unit}")
} else {
value.to_string()
};
let coordinate =
crate::plugins::model_coordinates::read(&typed, unit, engine, context)?;
Ok((coordinate.value, coordinate.authored))
})
.collect()
}
fn median(ordered: &[BigRational]) -> BigRational {
let middle = ordered.len() / 2;
if ordered.len().is_multiple_of(2) {
(&ordered[middle - 1] + &ordered[middle]) / BigRational::from_integer(2.into())
} else {
ordered[middle].clone()
}
}
fn dataset_report(input: &str, requested: &str, context: &Context) -> Result<String, Error> {
dataset_report_with_options(input, requested, None, false, context)
}
fn dataset_report_with_options(
input: &str,
requested: &str,
stated_count: Option<&str>,
integers: bool,
context: &Context,
) -> Result<String, Error> {
let input = list(input);
let values = coordinates(&input, context)?;
if let Some(stated) = stated_count {
let count = crate::plugins::integer_queries::value(stated)?;
if !count.is_positive() || count != num_bigint::BigInt::from(values.len()) {
return Err(error("invalid_dataset"));
}
}
if integers && values.iter().any(|(value, _)| !value.is_integer()) {
return Err(error("invalid_dataset"));
}
let authored = values.iter().all(|(_, authored)| *authored);
let mut ordered = values
.into_iter()
.map(|(value, _)| value)
.collect::<Vec<_>>();
ordered.sort();
let goals = dataset_goals(requested)?;
let mut seen = Vec::new();
let mut fields = Vec::new();
for goal in goals {
if seen.contains(&goal) {
return Err(error("invalid_dataset"));
}
seen.push(goal.clone());
let value = match goal.as_str() {
"count" => literal(&BigRational::from_integer(ordered.len().into()), true)?,
"sum" => literal(
&crate::plugins::model_coordinates::bounded(ordered.iter().cloned().sum())?,
authored,
)?,
"minimum" => literal(&ordered[0], authored)?,
"maximum" => literal(ordered.last().unwrap(), authored)?,
"mean" => {
let sum: BigRational = ordered.iter().cloned().sum();
literal(
&(sum / BigRational::from_integer(ordered.len().into())),
authored,
)?
}
"median" => literal(&median(&ordered), authored)?,
"range" => literal(&(ordered.last().unwrap() - &ordered[0]), authored)?,
"population variance"
| "sample variance"
| "population standard deviation"
| "sample standard deviation" => {
let operation = goal
.replace(" standard deviation", "_stddev")
.replace(" variance", "_variance");
let value = Probability
.compute(&format!("{operation}({input})"), context)
.ok_or_else(|| error("invalid_dataset"))??;
format!("real({value:e})")
}
"mode" => {
let mut frequencies: Vec<(&BigRational, usize)> = Vec::new();
for value in &ordered {
if let Some((previous, count)) = frequencies.last_mut()
&& *previous == value
{
*count += 1;
} else {
frequencies.push((value, 1));
}
}
let maximum = frequencies.iter().map(|(_, count)| *count).max().unwrap();
if maximum == 1 {
"none".into()
} else {
frequencies
.into_iter()
.filter(|(_, count)| *count == maximum)
.map(|(value, _)| literal(value, authored))
.collect::<Result<Vec<_>, _>>()?
.join(",")
}
}
_ => return Err(error("invalid_dataset")),
};
fields.push(format!("{goal}|{value}"));
}
report(fields)
}
fn percentage_coordinate(input: &str, context: &Context) -> Result<(BigRational, bool), Error> {
let c = re!(r"(?i)^(.+?)(?:%| percent)$")
.captures(input.trim())
.ok_or_else(|| error("invalid_probability_argument"))?;
let (value, authored) = coordinate(&c[1], context)?;
Ok((
crate::plugins::model_coordinates::bounded(value / BigRational::from_integer(100.into()))?,
authored,
))
}
fn portfolio(input: &str, context: &Context) -> Outcome {
let c = re!(r"(?i)^a two-asset portfolio holds (.+?) asset ([a-z][a-z0-9_]*) and (.+?) asset ([a-z][a-z0-9_]*); expected returns are (.+?), volatilities (?:are )?(.+?), and correlation (.+?)[.;] report (.+)$")
.captures(input)?;
Some((|| {
if c[2] == c[4] {
return Err(error("invalid_probability_argument"));
}
let bounded = crate::plugins::model_coordinates::bounded;
let weights = [
percentage_coordinate(&c[1], context)?,
percentage_coordinate(&c[3], context)?,
];
if weights.iter().any(|(weight, _)| weight.is_negative())
|| &weights[0].0 + &weights[1].0 != BigRational::one()
{
return Err(error("invalid_probability_argument"));
}
let returns = list(&c[5]);
let returns = split_arguments(&returns)?;
let volatilities = list(&c[6]);
let volatilities = split_arguments(&volatilities)?;
let ([return_a, return_b], [vol_a, vol_b]) = (returns.as_slice(), volatilities.as_slice())
else {
return Err(error("invalid_probability_argument"));
};
let returns = [
percentage_coordinate(return_a, context)?,
percentage_coordinate(return_b, context)?,
];
let volatilities = [
percentage_coordinate(vol_a, context)?,
percentage_coordinate(vol_b, context)?,
];
let (correlation, _) = coordinate(&c[7], context)?;
if volatilities
.iter()
.any(|(volatility, _)| volatility.is_negative())
|| correlation.abs() > BigRational::one()
{
return Err(error("invalid_probability_argument"));
}
let expectation = bounded(
bounded(&weights[0].0 * &returns[0].0)? + bounded(&weights[1].0 * &returns[1].0)?,
)?;
let a = bounded(&weights[0].0 * &volatilities[0].0)?;
let b = bounded(&weights[1].0 * &volatilities[1].0)?;
let variance = bounded(
bounded(&a * &a)?
+ bounded(&b * &b)?
+ bounded(BigRational::from_integer(2.into()) * correlation * &a * &b)?,
)?;
if variance.is_negative() {
return Err(error("invalid_probability_argument"));
}
let volatility = sqrt_ratio(
bounded(variance * BigRational::from_integer(10_000.into()))?,
context,
)?;
let expectation_authored = weights
.iter()
.chain(returns.iter())
.all(|(_, authored)| *authored);
let expectation = literal(
&bounded(expectation * BigRational::from_integer(100.into()))?,
expectation_authored,
)?;
let goals = list(&c[8]);
let goals = split_arguments(&goals)?;
if goals.len() != 2 {
return Err(error("invalid_dataset"));
}
let mut seen = Vec::new();
let mut fields = Vec::new();
for goal in goals {
let goal = goal.trim().to_ascii_lowercase();
let goal = goal.strip_prefix("the ").unwrap_or(&goal).to_owned();
if seen.contains(&goal) {
return Err(error("invalid_dataset"));
}
let value = match goal.as_str() {
"expected return" => expectation.clone(),
"portfolio volatility" => format!("real({volatility:e})"),
_ => return Err(error("invalid_dataset")),
};
fields.push(format!("{goal}|{value}"));
seen.push(goal);
}
report(fields)
})())
}
fn percentile_label(value: &BigRational) -> Result<String, Error> {
let number = value.to_f64().ok_or_else(|| error("invalid_dataset"))?;
let suffix = if value.is_integer() {
let n = value.to_u32().ok_or_else(|| error("invalid_dataset"))?;
match (n % 100, n % 10) {
(11..=13, _) => "th",
(_, 1) => "st",
(_, 2) => "nd",
(_, 3) => "rd",
_ => "th",
}
} else {
"th"
};
Ok(format!("{number}{suffix} percentile"))
}
fn interpolated_value(
ordered: &[BigRational],
percentile: &BigRational,
) -> Result<BigRational, Error> {
let bounded = crate::plugins::model_coordinates::bounded;
let rank = bounded(
BigRational::from_integer((ordered.len() - 1).into()) * percentile
/ BigRational::from_integer(100.into()),
)?;
let lower = rank
.to_integer()
.to_usize()
.ok_or_else(|| error("invalid_dataset"))?;
let fraction = rank - BigRational::from_integer(lower.into());
if fraction.is_zero() {
Ok(ordered[lower].clone())
} else {
bounded(&ordered[lower] + bounded(fraction * (&ordered[lower + 1] - &ordered[lower]))?)
}
}
fn nearest_rank_index(count: usize, percentile: &BigRational) -> Result<usize, Error> {
let rank = crate::plugins::model_coordinates::bounded(
BigRational::from_integer(count.into()) * percentile
/ BigRational::from_integer(100.into()),
)?;
rank.numer()
.div_ceil(rank.denom())
.to_usize()
.and_then(|rank| rank.checked_sub(1))
.filter(|index| *index < count)
.ok_or_else(|| error("invalid_dataset"))
}
fn two_method_percentile(input: &str, context: &Context) -> Outcome {
let c = re!(r"(?i)^the sorted dataset is (.+?)\.\s*report its (.+?)(?:st|nd|rd|th) percentile under both explicitly defined methods:\s*(.+)$")
.captures(input)?;
Some((|| {
let values = coordinates(&c[1], context)?;
if values.windows(2).any(|pair| pair[0].0 > pair[1].0) {
return Err(error("invalid_dataset"));
}
let (percentile, percentile_authored) = coordinate(&c[2], context)?;
if percentile <= BigRational::zero() || percentile > BigRational::from_integer(100.into()) {
return Err(error("invalid_probability_argument"));
}
let authored = percentile_authored && values.iter().all(|(_, authored)| *authored);
let ordered = values
.into_iter()
.map(|(value, _)| value)
.collect::<Vec<_>>();
let definitions = c[3].split(';').map(str::trim).collect::<Vec<_>>();
if definitions.len() != 2 {
return Err(error("invalid_dataset"));
}
let mut fields = Vec::new();
let mut seen = Vec::new();
for definition in definitions {
let (label, value) = if re!(r"(?i)^nearest[- ]rank uses rank (?:ceiling|ceil)\(\s*n\s*\*?\s*p\s*\)$").is_match(definition) {
("nearest-rank", ordered[nearest_rank_index(ordered.len(), &percentile)?].clone())
} else if re!(r"(?i)^interpolation uses h\s*=\s*1\s*\+\s*\(\s*n\s*-\s*1\s*\)\s*\*?\s*p with 1[- ]indexed positions$").is_match(definition) {
("interpolation", interpolated_value(&ordered, &percentile)?)
} else { return Err(error("invalid_dataset")); };
if seen.contains(&label) {
return Err(error("invalid_dataset"));
}
seen.push(label);
fields.push(format!("{label}|{}", literal(&value, authored)?));
}
report(fields)
})())
}
fn percentile_report(input: &str, context: &Context) -> Outcome {
let c = re!(r"(?i)^a dataset is (.+?); report (.+?) percentiles? using linear interpolation at rank\s*1\s*\+\s*\(\s*n\s*-\s*1\s*\)\s*\*?\s*p$")
.captures(input)?;
Some((|| {
let values = coordinates(&c[1], context)?;
let authored = values.iter().all(|(_, authored)| *authored);
let mut ordered = values
.into_iter()
.map(|(value, _)| value)
.collect::<Vec<_>>();
ordered.sort();
let goals = list(&c[2]);
let goals = split_arguments(&goals)?;
if !(1..=5).contains(&goals.len()) {
return Err(error("invalid_dataset"));
}
let hundred = BigRational::from_integer(100.into());
let mut seen = Vec::new();
let mut fields = Vec::new();
for goal in goals {
let c = re!(r"(?i)^(?:the )?(.+?)(?:st|nd|rd|th)$")
.captures(goal.trim())
.ok_or_else(|| error("invalid_dataset"))?;
let (percentile, percentile_authored) = coordinate(&c[1], context)?;
if percentile.is_negative() || percentile > hundred {
return Err(error("invalid_probability_argument"));
}
let label = percentile_label(&percentile)?;
if seen.contains(&label) {
return Err(error("invalid_dataset"));
}
let value = interpolated_value(&ordered, &percentile)?;
fields.push(format!(
"{label}|{}",
literal(&value, authored && percentile_authored)?
));
seen.push(label);
}
report(fields)
})())
}
fn nearest_rank(input: &str, context: &Context) -> Outcome {
let c = re!(r"(?i)^for\s+(?:data|dataset)\s+(.+?),\s*(?:compute|find)\s+p\s*(.+?)\s+under\s+the\s+nearest[- ]rank\s+(?:convention|method);\s*(?:state|give)\s+the\s+selected\s+value$")
.captures(input)?;
Some((|| {
let mut values = coordinates(&c[1], context)?;
let (percentile, _) = coordinate(
c[2].trim().strip_suffix('%').unwrap_or(c[2].trim()),
context,
)?;
let hundred = BigRational::from_integer(100.into());
if percentile <= BigRational::zero() || percentile > hundred {
return Err(error("invalid_probability_argument"));
}
values.sort_by(|a, b| a.0.cmp(&b.0));
let index = nearest_rank_index(values.len(), &percentile)?;
let (selected, authored) = &values[index];
let output = literal(selected, *authored)?;
if output.len() > 4096 {
return Err(error("invalid_dataset"));
}
Ok(output)
})())
}
pub(super) fn rewrite(input: &str, context: &Context) -> Outcome {
let s = input.trim().trim_end_matches(['.', '?']).replace('−', "-");
if let Some(c) = re!(r"(?is)^treat these values as frequencies paired in order with scores (.+?):\s*scores (.+?) and frequencies (.+?)\.\s*(?:report|give|state) (.+)$").captures(&s) {
return Some(if s.len() > 4096 {
Err(error("invalid_dataset"))
} else {
frequency_report(Some(&c[1]), &c[2], &c[3], &c[4], context)
});
}
if let Some(parts) = frequency_request(&s) {
return Some(if s.len() > 4096 {
Err(error("invalid_dataset"))
} else {
paired_frequency_report(parts, context)
});
}
if let Some(result) = descriptive_request(&s, context) {
return Some(result);
}
if s.len() > 4096 {
return None;
}
if let Some(result) = portfolio(&s, context)
.or_else(|| two_method_percentile(&s, context))
.or_else(|| percentile_report(&s, context))
.or_else(|| nearest_rank(&s, context))
{
return Some(result);
}
if let Some(c) = re!(r"(?i)^a fitted line is y\s*=\s*(.+?) and observed pairs are (.+?); report residuals in the listed order and their sum of squares$").captures(&s) {
return Some((|| {
let (intercept, coefficients) = crate::plugins::integer_queries::affine_value(&c[1], &["x"])?;
let [slope] = coefficients.as_slice() else {
return Err(error("invalid_dataset"));
};
let points = paired_points(&c[2], context)?;
if !(1..=256).contains(&points.len()) {
return Err(error("invalid_dataset"));
}
let authored = points.iter().all(|((_, x), (_, y))| *x && *y);
let (residuals, squares) = residuals(&points, slope, &intercept)?;
let values = residuals.iter().map(|value| literal(value, authored))
.collect::<Result<Vec<_>, _>>()?.join(",");
report(vec![format!("residuals|{values}"),
format!("residual sum of squares|{}", literal(&squares, authored)?)])
})());
}
if let Some(c) =
re!(r"(?i)^a discrete variable ([a-z][a-z0-9_]*) has probabilities (.+); report (.+)$")
.captures(&s)
{
return Some(pmf_report(&c[1], &c[2], &c[3], context));
}
if let Some(c) = re!(r"(?i)^given paired observations (.+), fit (?:the|an) ordinary least-squares line predicting y from x; report slope, intercept, and R-squared$").captures(&s) {
return Some((|| regression_report(&paired_points(&c[1], context)?, None))());
}
if let Some(c) = re!(r"(?i)^a [a-z][a-z ]*['’]s measured [a-z][a-z ]* y in (.+?) is (.+?) at positions x\s*=\s*(.+?); fit a least-squares line and report slope, intercept, and R-squared$").captures(&s) {
return Some((|| {
let engine = crate::plugins::compositions::Compositions::default();
let y_unit = axis_unit(&c[1], &engine)?;
let (values, x_unit) = c[3].char_indices().filter_map(|(at, _)| {
if at == 0 || !c[3][..at].ends_with(char::is_whitespace) {
return None;
}
axis_unit(c[3][at..].trim(), &engine).ok()
.map(|unit| (c[3][..at].trim(), unit))
}).next().ok_or_else(|| error("incompatible_units"))?;
let xs = axis_coordinates(values, &x_unit, &engine, context)?;
let ys = axis_coordinates(&c[2], &y_unit, &engine, context)?;
if xs.len() != ys.len() {
return Err(error("invalid_dataset"));
}
let points = xs.into_iter().zip(ys).collect::<Vec<_>>();
regression_report(&points, Some((&x_unit, &y_unit)))
})());
}
if let Some(c) = re!(r"(?i)^a time series has observations (.+); fit y\s*=\s*a\s*\+\s*b\s*t by least squares and predict day (.+), while also reporting the residual sum of squares$").captures(&s) {
return Some((|| {
let points_text = list(&c[1]);
let supplied = split_arguments(&points_text)?;
if !(2..=256).contains(&supplied.len()) {
return Err(error("invalid_dataset"));
}
let mut points = Vec::with_capacity(supplied.len());
for point in supplied {
let pair = point
.strip_prefix('(')
.and_then(|p| p.strip_suffix(')'))
.ok_or_else(|| error("invalid_dataset"))?;
let coordinates = split_arguments(pair)?;
let [day, value] = coordinates.as_slice() else {
return Err(error("invalid_dataset"));
};
let day = re!(r"(?i)^day\s+(.+)$")
.captures(day)
.ok_or_else(|| error("invalid_dataset"))?;
points.push((coordinate(&day[1], context)?, coordinate(value, context)?));
}
let (slope, intercept, authored) = least_squares(&points)?;
let (target, target_authored) = coordinate(&c[2], context)?;
let target_label = c[2].trim();
let prediction = crate::plugins::model_coordinates::bounded(
&intercept + crate::plugins::model_coordinates::bounded(&slope * &target)?,
)?;
let residual_squares = residual_squares(&points, &slope, &intercept)?;
report(vec![
format!("a|{}", literal(&intercept, authored)?),
format!("b|{}", literal(&slope, authored)?),
format!(
"prediction at day {target_label}|{}",
literal(&prediction, authored && target_authored)?
),
format!(
"residual sum of squares|{}",
literal(&residual_squares, authored)?
),
])
})());
}
if let Some(c) = re!(
r"(?i)^(?:calculate|find|compute) the least-squares slope and intercept for points (.+)$"
)
.captures(&s)
{
return Some((|| {
let points = paired_points(&c[1], context)?;
let (slope, intercept, authored) = least_squares(&points)?;
report(vec![
format!("slope|{}", literal(&slope, authored)?),
format!("y-intercept|{}", literal(&intercept, authored)?),
])
})());
}
if let Some(c) = re!(r"(?i)^given paired data x\s*=\s*\((.+?)\),\s*y\s*=\s*\((.+?)\),\s*estimate y at x\s*=\s*(.+?) using a least-squares line$").captures(&s) {
return Some((|| {
let xs = coordinates(&c[1], context)?;
let ys = coordinates(&c[2], context)?;
if xs.len() != ys.len() || xs.len() < 2 {
return Err(error("invalid_dataset"));
}
let points = xs.into_iter().zip(ys).collect::<Vec<_>>();
let (slope, intercept, authored) = least_squares(&points)?;
let (target, target_authored) = coordinate(&c[3], context)?;
let prediction = crate::plugins::model_coordinates::bounded(intercept + slope * target)?;
literal(&prediction, authored && target_authored)
})());
}
if let Some(c) = re!(r"(?i)^a sample of (.+?) has mean (.+?) and sample (?:sd|standard deviation) (.+?)\. using t critical (.+?) for (.+?) df, (?:calculate|find|report) the (.+?)% confidence interval for (?:the )?population mean, endpoints to two decimals$").captures(&s) {
return Some((|| {
let (n, _) = coordinate(&c[1], context)?;
let (mean, _) = coordinate(&c[2], context)?;
let (sd, _) = coordinate(&c[3], context)?;
let (critical, _) = coordinate(&c[4], context)?;
let (df, _) = coordinate(&c[5], context)?;
let (level, _) = coordinate(&c[6], context)?;
let one = BigRational::from_integer(1.into());
let hundred = BigRational::from_integer(100.into());
if !n.is_integer() || n < BigRational::from_integer(2.into())
|| !df.is_integer() || df != &n - &one
|| sd.is_negative() || critical <= BigRational::zero()
|| level <= BigRational::zero() || level >= hundred
{
return Err(error("invalid_dataset"));
}
let root = BigRational::from_float(sqrt_ratio(n, context)?)
.ok_or_else(|| error("numeric_overflow"))?;
let margin = crate::plugins::model_coordinates::bounded(critical * sd / root)?;
let lower = crate::plugins::model_coordinates::bounded(&mean - &margin)?;
let upper = crate::plugins::model_coordinates::bounded(mean + margin)?;
let expression = format!(
"statistics_interval({};{};{})",
literal(&level, true)?,
literal(&lower, false)?,
literal(&upper, false)?
);
if expression.len() > 4096 {
return Err(error("invalid_dataset"));
}
Ok(expression)
})());
}
if let Some(c) = re!(r"(?i)^for data values (.+?),\s*(?:find|calculate|report) (.+?) using the median-of-halves convention, excluding the overall median from each half when the count is odd$").captures(&s) {
return Some((|| {
let values = coordinates(&c[1], context)?;
if values.len() < 2 {
return Err(error("invalid_dataset"));
}
let authored = values.iter().all(|(_, exact)| *exact);
let mut ordered = values.into_iter().map(|(value, _)| value).collect::<Vec<_>>();
ordered.sort();
let middle = ordered.len() / 2;
let lower = median(&ordered[..middle]);
let upper = median(&ordered[ordered.len().div_ceil(2)..]);
let median_value = median(&ordered);
let range = crate::plugins::model_coordinates::bounded(upper - lower)?;
let goals = list(&c[2]);
let goals = split_arguments(&goals)?;
if goals.len() != 2 {
return Err(error("invalid_dataset"));
}
let mut fields = Vec::with_capacity(2);
for goal in goals {
let goal = goal.trim().to_ascii_lowercase();
let goal = goal.strip_prefix("the ").unwrap_or(&goal);
if fields.iter().any(|field: &String| field.starts_with(&format!("{goal}|"))) {
return Err(error("invalid_dataset"));
}
let value = match goal {
"median" => literal(&median_value, authored)?,
"interquartile range" => literal(&range, authored)?,
_ => return Err(error("invalid_dataset")),
};
fields.push(format!("{goal}|{value}"));
}
report(fields)
})());
}
if let Some(c) = re!(
r"(?i)^for (?:the )?values (.+?), calculate the sum of absolute deviations from the median$"
)
.captures(&s)
{
return Some((|| {
let mut values = coordinates(&c[1], context)?;
let authored = values.iter().all(|(_, authored)| *authored);
let mut ordered = values.drain(..).map(|(value, _)| value).collect::<Vec<_>>();
ordered.sort();
let median = median(&ordered);
let total = ordered
.iter()
.map(|value| (value - &median).abs())
.sum::<BigRational>();
literal(
&crate::plugins::model_coordinates::bounded(total)?,
authored,
)
})());
}
if let Some(c) = re!(r"(?i)^for paired observations x=\((.+?)\) and y=\((.+?)\), what is the pearson correlation coefficient$").captures(&s) {
return Some((|| {
let xs = coordinates(&c[1], context)?;
let ys = coordinates(&c[2], context)?;
if xs.len() != ys.len() || xs.len() < 2 {
return Err(error("invalid_dataset"));
}
let count = BigRational::from_integer(xs.len().into());
let mean_x = xs.iter().map(|(value, _)| value.clone()).sum::<BigRational>() / &count;
let mean_y = ys.iter().map(|(value, _)| value.clone()).sum::<BigRational>() / count;
let (mut cross, mut square_x, mut square_y) =
(BigRational::zero(), BigRational::zero(), BigRational::zero());
for ((x, _), (y, _)) in xs.iter().zip(&ys) {
let dx = x - &mean_x;
let dy = y - &mean_y;
cross += &dx * &dy;
square_x += &dx * &dx;
square_y += &dy * &dy;
}
if square_x.is_zero() || square_y.is_zero() {
return Err(error("invalid_dataset"));
}
let squared = crate::plugins::model_coordinates::bounded(
&cross * &cross / (square_x * square_y),
)?;
let ratio = sqrt_ratio(squared, context)?;
Ok(format!("real({:e})", if cross.is_negative() { -ratio } else { ratio }))
})());
}
if let Some(c) = re!(r"(?i)^a sample has mean (.+?) and standard deviation (.+?); calculate the z-score of value (.+)$").captures(&s) {
return Some((|| {
let (mean, ma) = coordinate(&c[1], context)?;
let (sd, sa) = coordinate(&c[2], context)?;
let (value, va) = coordinate(&c[3], context)?;
if sd <= BigRational::zero() {
return Err(error("invalid_dataset"));
}
literal(&((value - mean) / sd), ma && sa && va)
})());
}
if let Some(c) = re!(
r"(?i)^for (scores|(?:the )?(?:values|data)) (.+?), (?:calculate|find|report|compute) (.+)$"
)
.captures(&s)
{
if c[1].eq_ignore_ascii_case("scores") || dataset_goals(&c[3]).is_ok() {
return Some(dataset_report(&c[2], &c[3], context));
}
}
if let Some(c) = re!(r"(?i)^given the measurements (.+?), (?:calculate|find|compute) mean absolute deviation from the mean$").captures(&s) {
return Some((|| {
let coordinates = coordinates(&c[1], context)?;
let count = BigRational::from_integer(coordinates.len().into());
let mean = coordinates.iter().map(|(value, _)| value.clone()).sum::<BigRational>() / &count;
let deviation = coordinates.iter().map(|(value, _)| (value - &mean).abs()).sum::<BigRational>() / count;
literal(&deviation, coordinates.iter().all(|(_, authored)| *authored))
})());
}
if let Some(c) =
re!(r"(?i)^a course grade uses (.+?); (?:calculate|find|compute) the weighted grade$")
.captures(&s)
{
return Some((|| {
let supplied = list(&c[1]);
let pieces = split_arguments(&supplied)?;
if pieces.is_empty() || pieces.len() > 256 {
return Err(error("invalid_weights"));
}
let mut values = Vec::new();
let mut weights = Vec::new();
let mut labels = Vec::new();
let mut sum = BigRational::zero();
let hundred = BigRational::from_integer(100.into());
for piece in pieces {
let pair = re!(r"(?i)^([a-z][a-z ]*?) (.+?) at weight (.+?)%$")
.captures(piece)
.ok_or_else(|| error("invalid_weights"))?;
let label = pair[1].to_ascii_lowercase();
if labels.contains(&label) {
return Err(error("invalid_weights"));
}
labels.push(label);
coordinate(&pair[2], context)?;
let (weight, _) = coordinate(&pair[3], context)?;
if weight.is_negative() || weight > hundred {
return Err(error("invalid_weights"));
}
sum += weight;
values.push(pair[2].to_string());
weights.push(format!("({})/100", &pair[3]));
}
if sum != hundred {
return Err(error("invalid_weights"));
}
Ok(format!(
"weighted_mean([{}],[{}])",
values.join(","),
weights.join(",")
))
})());
}
if let Some(c) = re!(r"(?i)^a data set has quartiles q1=(.+?) and q3=(.+?); report its interquartile range and the fences q1-1\.5 iqr and q3\+1\.5 iqr$").captures(&s) {
return Some((|| {
let (q1, first_authored) = coordinate(&c[1], context)?;
let (q3, third_authored) = coordinate(&c[2], context)?;
if q1 > q3 { return Err(error("invalid_dataset")); }
let iqr = &q3 - &q1; let fence = &iqr * BigRational::new(3.into(), 2.into());
let authored = first_authored && third_authored;
report(vec![
format!("interquartile range|{}", literal(&iqr, authored)?),
format!("lower fence|{}", literal(&(q1 - &fence), authored)?),
format!("upper fence|{}", literal(&(q3 + fence), authored)?),
])
})());
}
if let Some(c) = re!(r"(?i)^a survey has (.+?) positive answers out of (.+?); report the proportion, percentage, and percentage of nonpositive answers$").captures(&s) {
return Some((|| {
let (positive, positive_authored) = coordinate(&c[1], context)?;
let (total, total_authored) = coordinate(&c[2], context)?;
if !positive.is_integer() || !total.is_integer() || positive.is_negative() || total.is_negative() || positive > total {
return Err(error("invalid_population"));
}
if total.is_zero() { return Err(error("division_by_zero")); }
let ratio = positive / total;
let authored = positive_authored && total_authored;
report(vec![
format!("proportion|{}", literal(&ratio, authored)?),
format!("positive share|{}", literal(&(&ratio * BigRational::from_integer(100.into())), authored)?),
format!("nonpositive share|{}", literal(&((BigRational::from_integer(1.into()) - ratio) * BigRational::from_integer(100.into())), authored)?),
])
})());
}
if let Some(c) =
re!(r"(?i)^(?:the values (.+?) are a data set|a data set is (.+?)); (?:report|find|calculate|compute) (?:its )?(.+)$").captures(&s)
{
return Some(dataset_report(c.get(1).or_else(|| c.get(2)).unwrap().as_str(), &c[3], context));
}
if let Some(c) = re!(r"(?i)^for the measurements (.+?), (?:report|find|calculate) (.+?)(?: so i can compare their sensitivity to the outlier)?$").captures(&s) {
return Some(dataset_report(&c[1], &c[2], context));
}
if let Some(c) = re!(r"(?i)^for (?:scores|the values) (.+?), (?:find|report|calculate|compute) the (population|sample) standard deviation and the (population|sample) standard deviation$").captures(&s) {
return Some(dataset_report(&c[1], &format!("{} standard deviation, {} standard deviation", &c[2], &c[3]), context));
}
if let Some(c) = re!(r"(?i)^a test score of (.+?) is (.+?) standard deviations (above|below) a mean of (.+?); find the standard deviation$").captures(&s) {
return Some((|| {
let (score, score_authored) = coordinate(&c[1], context)?;
let (count, count_authored) = coordinate(&c[2], context)?;
let (mean, mean_authored) = coordinate(&c[4], context)?;
let distance = if c[3].eq_ignore_ascii_case("above") { score - mean } else { mean - score };
if count <= BigRational::zero() || distance <= BigRational::zero() {
return Err(error("invalid_dataset"));
}
literal(&(distance / count), score_authored && count_authored && mean_authored)
})());
}
if let Some(c) = re!(r"(?i)^(?:calculate|find|compute) the weighted mean of (.+)$").captures(&s)
{
if c[1].to_ascii_lowercase().contains("with weight ") {
return Some((|| {
let data = list(&c[1]);
let mut values = Vec::new();
let mut weights = Vec::new();
for part in split_arguments(&data)? {
let pair = re!(r"(?i)^(.+?) with weight (.+)$")
.captures(part)
.ok_or_else(|| error("invalid_weights"))?;
values.push(pair[1].to_string());
weights.push(pair[2].to_string());
}
Ok(format!(
"weighted_mean([{}],[{}])",
values.join(","),
weights.join(",")
))
})());
}
}
if let Some(c) = re!(r"(?i)^from a normal distribution with mean (.+?) and standard deviation (.+?), how many standard deviations (above|below) the mean is a score of (.+)$").captures(&s) {
return Some((|| {
scalar(&c[1], context)?; scalar(&c[4], context)?;
let (sd, _) = coordinate(&c[2], context)?;
if sd <= BigRational::zero() { return Err(error("invalid_dataset")); }
let expression = if c[3].eq_ignore_ascii_case("below") { format!("(({})-({}))/({})", &c[1], &c[4], &c[2]) }
else { format!("(({})-({}))/({})", &c[4], &c[1], &c[2]) };
scalar(&expression, context)?;
Ok(expression)
})());
}
None
}
pub(super) fn evaluate(input: &str, context: &Context) -> Outcome {
if let Some(inner) = input
.strip_prefix("statistics_interval(")
.and_then(|s| s.strip_suffix(')'))
{
return Some((|| {
if input.len() > 4096 {
return Err(error("invalid_dataset"));
}
let fields = inner.split(';').collect::<Vec<_>>();
let [level, lower, upper] = fields.as_slice() else {
return Err(error("invalid_dataset"));
};
let (level, _) = coordinate(level, context)?;
let lower = scalar(lower, context)?;
let upper = scalar(upper, context)?;
if level <= BigRational::zero()
|| level >= BigRational::from_integer(100.into())
|| lower > upper
{
return Err(error("invalid_dataset"));
}
let level = level.to_f64().ok_or_else(|| error("numeric_overflow"))?;
Ok(format!(
"{}% confidence interval: [{lower:.2}, {upper:.2}]",
crate::number(level)
))
})());
}
let fields = input
.strip_prefix("statistics_report(")?
.strip_suffix(')')?;
Some((|| {
let fields = fields.split(';').collect::<Vec<_>>();
if input.len() > 4096 || !report_fields(&fields) {
return Err(error("invalid_dataset"));
}
let engine = crate::plugins::compositions::Compositions::default();
let mut seen = Vec::new();
let mut output = Vec::new();
for field in fields {
let parts = field.split('|').collect::<Vec<_>>();
let (label, expression, reference) = match parts.as_slice() {
[label, expression] => (*label, *expression, None),
[label @ ("slope" | "y-intercept"), expression, unit] => {
(*label, *expression, Some(axis_unit(unit, &engine)?))
}
_ => return Err(error("invalid_dataset")),
};
let known_label = matches!(
label,
"a" | "b"
| "residual sum of squares"
| "residuals"
| "R-squared"
| "count"
| "sum"
| "minimum"
| "maximum"
| "mean"
| "slope"
| "y-intercept"
| "median"
| "mode"
| "range"
| "population standard deviation"
| "sample standard deviation"
| "population variance"
| "sample variance"
| "interquartile range"
| "lower fence"
| "upper fence"
| "total frequency"
| "weighted mean"
| "weighted median"
| "weighted population variance"
| "expected return"
| "portfolio volatility"
| "proportion"
| "positive share"
| "nonpositive share"
| "nearest-rank"
| "interpolation"
) || re!(
r"^[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?(?:st|nd|rd|th) percentile$"
)
.is_match(label)
&& label.split_once(' ').is_some_and(|(ordinal, _)| {
coordinate(&ordinal[..ordinal.len() - 2], context).is_ok_and(|(value, _)| {
!value.is_negative() && value <= BigRational::from_integer(100.into())
})
})
|| pmf_goal(label).is_some_and(|goal| match goal {
PmfGoal::AtLeast(_, threshold) => coordinate(threshold, context).is_ok(),
_ => true,
})
|| label
.strip_prefix("prediction at day ")
.is_some_and(|day| coordinate(day, context).is_ok());
if !known_label || seen.contains(&label) {
return Err(error("invalid_dataset"));
}
seen.push(label);
let value = if label == "mode" && expression == "none" {
"none".into()
} else {
let pieces = split_arguments(expression)?;
if pieces.is_empty()
|| label == "residuals" && pieces.len() > 256
|| !matches!(label, "mode" | "residuals") && pieces.len() != 1
{
return Err(error("invalid_dataset"));
}
pieces
.into_iter()
.map(|value| {
let coefficient = engine.quantity_value(value, "1", context)?;
if let Some(unit) = &reference {
engine.quantity_value(
&format!("{coefficient:e} {unit}"),
unit,
context,
)?;
}
if value.starts_with("rational(") {
let (n, d) = crate::plugins::integer_queries::ratio_value(value)?;
if d == 1.into() {
return Ok(crate::plugins::integer_queries::decimal(&n));
}
}
engine
.evaluate(value, context)
.ok_or_else(|| error("invalid_dataset"))?
})
.collect::<Result<Vec<_>, _>>()?
.join(", ")
};
let value = if label == "residuals" {
format!("[{value}]")
} else {
value
};
let (label, unit) = match label {
"expected return" | "portfolio volatility" => (label, "%"),
"positive share" => ("percentage", "%"),
"nonpositive share" => ("nonpositive percentage", "%"),
_ => (label, ""),
};
let reference = reference.map(|unit| format!(" {unit}")).unwrap_or_default();
output.push(format!("{label}: {value}{unit}{reference}"));
}
Ok(output.join("; "))
})())
}