Skip to main content

weirdnet_code_spaces/
math.rs

1//! Math code spaces: one formula per line, compiled to Rust over columns.
2//!
3//! ```text
4//! # moving averages on the input bars, per symbol (group the input by symbol)
5//! @title SMA cross
6//! @show close, fast, slow
7//! fast = sma(close, 10)
8//! slow = sma(close, 30)
9//! cross = if(fast > slow, 1, 0)
10//! @update signal = cross
11//! ```
12//!
13//! A name is an input field or an earlier line; every value is a column (one number per
14//! record, `NaN` when missing) computed per group of the input. See `docs/code-spaces.md`.
15use crate::Profile;
16use crate::error::{Error, Result};
17use std::collections::BTreeSet;
18
19/// Settings shared by math, Polars and Python sources (`@name value` lines).
20#[derive(Debug, Clone, Default, PartialEq)]
21pub struct Directives {
22    pub title: String,
23    pub view: String,
24    pub x: String,
25    pub show: Vec<String>,
26    pub updates: Vec<(String, String)>,
27}
28
29impl Directives {
30    /// Read a directive (`title …`, `view chart`, …) without its `@`.
31    pub fn read(&mut self, line: &str) -> std::result::Result<(), String> {
32        let (k, v) = line.split_once(char::is_whitespace).unwrap_or((line, ""));
33        let v = v.trim();
34        match k.to_ascii_lowercase().as_str() {
35            "title" => self.title = v.to_string(),
36            "view" => match v {
37                "chart" | "table" | "value" => self.view = v.to_string(),
38                _ => return Err(format!("@view {v}: use chart, table or value")),
39            },
40            "x" => {
41                ident(v)?;
42                self.x = v.to_string()
43            }
44            "show" => {
45                for n in v.split(',').map(str::trim).filter(|s| !s.is_empty()) {
46                    ident(n)?;
47                    self.show.push(n.to_string());
48                }
49            }
50            "update" => {
51                let (f, c) = v.split_once('=').ok_or("@update FIELD = NAME")?;
52                let (f, c) = (f.trim(), c.trim());
53                ident(f)?;
54                ident(c)?;
55                self.updates.push((f.to_string(), c.to_string()));
56            }
57            _ => return Err(format!("unknown @{k} (title, view, x, show, update)")),
58        }
59        Ok(())
60    }
61
62    /// The SDK's `columns::Output` for these settings (`sdk` is the SDK's crate name).
63    pub fn output_rs(&self, sdk: &str) -> String {
64        let list = |v: &[String]| v.iter().map(|s| format!("{s:?}.to_string()")).collect::<Vec<_>>().join(", ");
65        let ups = self.updates.iter().map(|(f, c)| format!("({f:?}.to_string(), {c:?}.to_string())")).collect::<Vec<_>>().join(", ");
66        format!(
67            "{sdk}::columns::Output {{ title: {:?}.to_string(), view: {:?}.to_string(), x: {:?}.to_string(), show: vec![{}], updates: vec![{ups}] }}",
68            self.title,
69            self.view,
70            self.x,
71            list(&self.show)
72        )
73    }
74}
75
76pub(crate) fn ident(s: &str) -> std::result::Result<(), String> {
77    let mut c = s.chars();
78    let ok = c.next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_') && c.all(|c| c.is_ascii_alphanumeric() || c == '_');
79    if ok { Ok(()) } else { Err(format!("`{s}` is not a name (letters, digits and _)")) }
80}
81
82#[derive(Debug, Clone, PartialEq)]
83enum Tok {
84    Num(f64),
85    Id(String),
86    Op(&'static str),
87    L,
88    R,
89    Comma,
90}
91
92fn lex(s: &str) -> std::result::Result<Vec<Tok>, String> {
93    let cs: Vec<char> = s.chars().collect();
94    let mut i = 0;
95    let mut out = vec![];
96    const OPS: [&str; 18] = ["<=", ">=", "==", "!=", "&&", "||", "+", "-", "*", "/", "%", "^", "<", ">", "!", "=", "&", "|"];
97    while i < cs.len() {
98        let c = cs[i];
99        if c.is_whitespace() {
100            i += 1;
101        } else if c.is_ascii_digit() || (c == '.' && cs.get(i + 1).is_some_and(|d| d.is_ascii_digit())) {
102            let st = i;
103            while i < cs.len() && (cs[i].is_ascii_digit() || cs[i] == '.' || cs[i] == '_') {
104                i += 1;
105            }
106            if i < cs.len() && (cs[i] == 'e' || cs[i] == 'E') {
107                i += 1;
108                if i < cs.len() && (cs[i] == '+' || cs[i] == '-') {
109                    i += 1;
110                }
111                while i < cs.len() && cs[i].is_ascii_digit() {
112                    i += 1;
113                }
114            }
115            let t: String = cs[st..i].iter().filter(|c| **c != '_').collect();
116            let v: f64 = t.parse().map_err(|_| format!("`{t}` is not a number"))?;
117            if !v.is_finite() {
118                return Err(format!("`{t}` is too large"));
119            }
120            out.push(Tok::Num(v));
121        } else if c.is_ascii_alphabetic() || c == '_' {
122            let st = i;
123            let mut dotted = false;
124            while i < cs.len() && (cs[i].is_ascii_alphanumeric() || cs[i] == '_') {
125                i += 1;
126                // `INPUT.field`: a field of a named input (a pipeline's fan-in)
127                if !dotted && i + 1 < cs.len() && cs[i] == '.' && (cs[i + 1].is_ascii_alphabetic() || cs[i + 1] == '_') {
128                    dotted = true;
129                    i += 1;
130                }
131            }
132            out.push(match cs[st..i].iter().collect::<String>().as_str() {
133                "and" => Tok::Op("&&"),
134                "or" => Tok::Op("||"),
135                "not" => Tok::Op("!"),
136                w => Tok::Id(w.to_string()),
137            });
138        } else if c == '(' {
139            out.push(Tok::L);
140            i += 1;
141        } else if c == ')' {
142            out.push(Tok::R);
143            i += 1;
144        } else if c == ',' {
145            out.push(Tok::Comma);
146            i += 1;
147        } else {
148            let two: String = cs[i..(i + 2).min(cs.len())].iter().collect();
149            let op = OPS.iter().find(|o| two.starts_with(**o)).ok_or_else(|| format!("unexpected `{c}`"))?;
150            i += op.len();
151            out.push(Tok::Op(match *op {
152                "=" => "==",
153                "&" => "&&",
154                "|" => "||",
155                o => o,
156            }));
157        }
158    }
159    Ok(out)
160}
161
162/// An expression.
163#[derive(Debug, Clone, PartialEq)]
164pub enum Expr {
165    Num(f64),
166    Name(String),
167    Neg(Box<Expr>),
168    Not(Box<Expr>),
169    Bin(&'static str, Box<Expr>, Box<Expr>),
170    Call(String, Vec<Expr>),
171}
172
173struct P {
174    t: Vec<Tok>,
175    i: usize,
176}
177
178fn prec(op: &str) -> Option<(u8, bool)> {
179    Some(match op {
180        "||" => (1, false),
181        "&&" => (2, false),
182        "<" | "<=" | ">" | ">=" | "==" | "!=" => (4, false),
183        "+" | "-" => (5, false),
184        "*" | "/" | "%" => (6, false),
185        "^" => (8, true),
186        _ => return None,
187    })
188}
189
190impl P {
191    fn peek(&self) -> Option<&Tok> {
192        self.t.get(self.i)
193    }
194    fn next(&mut self) -> Option<Tok> {
195        let t = self.t.get(self.i).cloned();
196        self.i += 1;
197        t
198    }
199    fn expr(&mut self, min: u8) -> std::result::Result<Expr, String> {
200        let mut lhs = match self.next() {
201            Some(Tok::Num(n)) => Expr::Num(n),
202            Some(Tok::Op("-")) => Expr::Neg(Box::new(self.expr(7)?)),
203            Some(Tok::Op("+")) => self.expr(7)?,
204            Some(Tok::Op("!")) => Expr::Not(Box::new(self.expr(3)?)),
205            Some(Tok::L) => {
206                let e = self.expr(0)?;
207                if self.next() != Some(Tok::R) {
208                    return Err("missing `)`".into());
209                }
210                e
211            }
212            Some(Tok::Id(n)) => {
213                if self.peek() == Some(&Tok::L) {
214                    self.i += 1;
215                    let mut args = vec![];
216                    if self.peek() == Some(&Tok::R) {
217                        self.i += 1;
218                    } else {
219                        loop {
220                            args.push(self.expr(0)?);
221                            match self.next() {
222                                Some(Tok::Comma) => {}
223                                Some(Tok::R) => break,
224                                _ => return Err(format!("{n}(…): missing `)` or `,`")),
225                            }
226                        }
227                    }
228                    Expr::Call(n, args)
229                } else {
230                    Expr::Name(n)
231                }
232            }
233            Some(t) => return Err(format!("unexpected {}", tok_text(&t))),
234            None => return Err("the formula ends too soon".into()),
235        };
236        while let Some(Tok::Op(op)) = self.peek().cloned() {
237            let Some((p, right)) = prec(op) else { return Err(format!("unexpected `{op}`")) };
238            if p < min {
239                break;
240            }
241            self.i += 1;
242            let rhs = self.expr(if right { p } else { p + 1 })?;
243            lhs = Expr::Bin(op, Box::new(lhs), Box::new(rhs));
244        }
245        Ok(lhs)
246    }
247}
248
249fn tok_text(t: &Tok) -> String {
250    match t {
251        Tok::Num(n) => n.to_string(),
252        Tok::Id(s) => format!("`{s}`"),
253        Tok::Op(o) => format!("`{o}`"),
254        Tok::L => "`(`".into(),
255        Tok::R => "`)`".into(),
256        Tok::Comma => "`,`".into(),
257    }
258}
259
260pub fn parse_expr(s: &str) -> std::result::Result<Expr, String> {
261    let mut p = P { t: lex(s)?, i: 0 };
262    let e = p.expr(0)?;
263    if let Some(t) = p.peek() {
264        return Err(format!("unexpected {} after the formula", tok_text(t)));
265    }
266    Ok(e)
267}
268
269/// Functions: (name, arguments, Rust over columns with `{0}`, `{1}`, `{2}` as `&[f64]`).
270const FUNCS: &[(&str, usize, &str)] = &[
271    ("sma", 2, "s::sma({0}, {1})"),
272    ("ema", 2, "s::ema({0}, {1})"),
273    ("rolling_sum", 2, "s::rolling_sum({0}, {1})"),
274    ("rolling_min", 2, "s::rolling_min({0}, {1})"),
275    ("rolling_max", 2, "s::rolling_max({0}, {1})"),
276    ("rolling_std", 2, "s::rolling_std({0}, {1})"),
277    ("lag", 2, "s::lag({0}, {1})"),
278    ("lead", 2, "s::lead({0}, {1})"),
279    ("diff", 1, "s::diff({0})"),
280    ("pct_change", 1, "s::pct_change({0})"),
281    ("cumsum", 1, "s::cumsum({0})"),
282    ("zscore", 1, "s::zscore({0})"),
283    ("sum", 1, "s::sum({0})"),
284    ("mean", 1, "s::mean({0})"),
285    ("avg", 1, "s::mean({0})"),
286    ("min", 1, "s::min({0})"),
287    ("max", 1, "s::max({0})"),
288    ("std", 1, "s::std({0})"),
289    ("count", 1, "s::count({0})"),
290    ("first", 1, "s::first({0})"),
291    ("last", 1, "s::last({0})"),
292    ("min", 2, "s::zip({0}, {1}, f64::min)"),
293    ("max", 2, "s::zip({0}, {1}, f64::max)"),
294    ("pow", 2, "s::zip({0}, {1}, f64::powf)"),
295    ("coalesce", 2, "s::zip({0}, {1}, |x, y| if x.is_nan() { y } else { x })"),
296    ("if", 3, "s::choose({0}, {1}, {2})"),
297    ("abs", 1, "s::map({0}, f64::abs)"),
298    ("sqrt", 1, "s::map({0}, f64::sqrt)"),
299    ("ln", 1, "s::map({0}, f64::ln)"),
300    ("log", 1, "s::map({0}, f64::ln)"),
301    ("log10", 1, "s::map({0}, f64::log10)"),
302    ("exp", 1, "s::map({0}, f64::exp)"),
303    ("round", 1, "s::map({0}, f64::round)"),
304    ("floor", 1, "s::map({0}, f64::floor)"),
305    ("ceil", 1, "s::map({0}, f64::ceil)"),
306    ("sign", 1, "s::map({0}, |x| if x.is_nan() || x == 0.0 { x } else { x.signum() })"),
307    ("missing", 1, "s::map({0}, |x| s::flag(x.is_nan()))"),
308];
309
310/// The function names (for help).
311pub fn functions() -> Vec<String> {
312    let mut v: Vec<String> = FUNCS.iter().map(|(n, a, _)| format!("{n}/{a}")).collect();
313    v.dedup();
314    v
315}
316
317/// A parsed math source.
318#[derive(Debug, Clone, Default, PartialEq)]
319pub struct Program {
320    /// (name, formula) in order.
321    pub lines: Vec<(String, Expr)>,
322    pub dirs: Directives,
323}
324
325impl Program {
326    /// Input fields the formulas read (names that are not lines).
327    pub fn fields(&self) -> BTreeSet<String> {
328        let mut defined = BTreeSet::new();
329        let mut used = BTreeSet::new();
330        for (n, e) in &self.lines {
331            names(e, &defined, &mut used);
332            defined.insert(n.clone());
333        }
334        let mut more = vec![self.dirs.x.clone()];
335        more.extend(self.dirs.show.iter().cloned());
336        more.extend(self.dirs.updates.iter().map(|(_, c)| c.clone()));
337        for m in more.into_iter().filter(|m| !m.is_empty() && !defined.contains(m)) {
338            used.insert(m);
339        }
340        used
341    }
342}
343
344fn names(e: &Expr, defined: &BTreeSet<String>, out: &mut BTreeSet<String>) {
345    match e {
346        Expr::Num(_) => {}
347        Expr::Name(n) => {
348            if !defined.contains(n) {
349                out.insert(n.clone());
350            }
351        }
352        Expr::Neg(a) | Expr::Not(a) => names(a, defined, out),
353        Expr::Bin(_, a, b) => {
354            names(a, defined, out);
355            names(b, defined, out);
356        }
357        Expr::Call(_, args) => args.iter().for_each(|a| names(a, defined, out)),
358    }
359}
360
361/// Parse a math source (lines `name = formula`, `@directive …`, `# comment`).
362pub fn parse(src: &str) -> Result<Program> {
363    parse_with(src, "")
364}
365
366/// [`parse`] with the profile's help hint after an unknown function.
367pub fn parse_for(profile: &Profile, src: &str) -> Result<Program> {
368    parse_with(src, profile.math_help)
369}
370
371fn parse_with(src: &str, help: &str) -> Result<Program> {
372    let mut p = Program::default();
373    let mut defined = BTreeSet::new();
374    for (i, raw) in src.lines().enumerate() {
375        let line = raw.split_once('#').map(|(a, _)| a).unwrap_or(raw).trim();
376        let err = |m: String| Error::invalid("math", format!("line {}: {m}", i + 1));
377        if line.is_empty() {
378            continue;
379        }
380        if let Some(d) = line.strip_prefix('@') {
381            p.dirs.read(d).map_err(err)?;
382            continue;
383        }
384        let Some((n, f)) = line.split_once('=').filter(|(_, f)| !f.starts_with('=')) else {
385            return Err(err("write NAME = FORMULA (or @title, @view, @x, @show, @update)".into()));
386        };
387        let n = n.trim();
388        ident(n).map_err(err)?;
389        if FUNCS.iter().any(|(f, _, _)| *f == n) {
390            return Err(err(format!("`{n}` is a function; pick another name")));
391        }
392        if defined.contains(n) {
393            return Err(err(format!("`{n}` is already defined")));
394        }
395        let e = parse_expr(f).map_err(err)?;
396        check(&e, help).map_err(err)?;
397        defined.insert(n.to_string());
398        p.lines.push((n.to_string(), e));
399    }
400    if p.lines.is_empty() {
401        return Err(Error::invalid("math", "no formulas: write NAME = FORMULA lines, e.g. `fast = sma(close, 10)`"));
402    }
403    for (_, c) in &p.dirs.updates {
404        if !defined.contains(c) {
405            return Err(Error::invalid("math", format!("@update: `{c}` is not one of the formulas")));
406        }
407    }
408    Ok(p)
409}
410
411fn check(e: &Expr, help: &str) -> std::result::Result<(), String> {
412    match e {
413        Expr::Call(n, args) => {
414            if !FUNCS.iter().any(|(f, a, _)| f == n && *a == args.len()) {
415                let arities: Vec<String> = FUNCS.iter().filter(|(f, _, _)| f == n).map(|(_, a, _)| a.to_string()).collect();
416                return Err(if arities.is_empty() {
417                    format!("unknown function `{n}`{help}")
418                } else {
419                    format!("{n} takes {} argument(s), not {}", arities.join(" or "), args.len())
420                });
421            }
422            args.iter().try_for_each(|a| check(a, help))
423        }
424        Expr::Neg(a) | Expr::Not(a) => check(a, help),
425        Expr::Bin(_, a, b) => check(a, help).and(check(b, help)),
426        _ => Ok(()),
427    }
428}
429
430fn rs(e: &Expr, defined: &BTreeSet<String>) -> String {
431    let r = |e: &Expr| format!("&{}", rs(e, defined));
432    match e {
433        Expr::Num(n) => format!("s::k({n:?}, n)"),
434        Expr::Name(x) if defined.contains(x) => format!("v_{x}.clone()"),
435        Expr::Name(x) => format!("f_{}.clone()", var(x)),
436        Expr::Neg(a) => format!("s::map({}, |x| -x)", r(a)),
437        Expr::Not(a) => format!("s::map({}, |x| s::flag(!s::truthy(x)))", r(a)),
438        Expr::Bin(op, a, b) => {
439            let f = match *op {
440                "+" | "-" | "*" | "/" | "%" => format!("|x, y| x {op} y"),
441                "^" => "f64::powf".into(),
442                "&&" => "|x, y| s::flag(s::truthy(x) && s::truthy(y))".into(),
443                "||" => "|x, y| s::flag(s::truthy(x) || s::truthy(y))".into(),
444                cmp => format!("|x, y| s::flag(x {cmp} y)"),
445            };
446            format!("s::zip({}, {}, {f})", r(a), r(b))
447        }
448        Expr::Call(n, args) => {
449            let tpl = FUNCS.iter().find(|(f, a, _)| f == n && *a == args.len()).map(|x| x.2).unwrap_or("s::k(f64::NAN, n)");
450            let mut out = tpl.to_string();
451            for (i, a) in args.iter().enumerate() {
452                out = out.replace(&format!("{{{i}}}"), &r(a));
453            }
454            out
455        }
456    }
457}
458
459/// A name as a Rust identifier (`slow.close` → `slow__close`).
460fn var(x: &str) -> String {
461    x.replace('.', "__")
462}
463
464/// The Rust `user.rs` for a math source, against the profile's SDK.
465pub fn generate(profile: &Profile, src: &str) -> Result<String> {
466    let p = parse_for(profile, src)?;
467    let mut defined = BTreeSet::new();
468    let mut body = String::new();
469    let mut named = String::new();
470    for f in p.fields() {
471        match f.split_once('.') {
472            // a named input's column, row by row beside the first input's rows (NaN past its end)
473            Some((input, field)) => {
474                let v = var(&f);
475                named += &format!("    let n_{v} = host.input_named({input:?})?.f64({field:?})?;\n");
476                body += &format!(
477                    "        let f_{v}: Vec<f64> = (grp.start..grp.start + n).map(|i| n_{v}.get(i).copied().unwrap_or(f64::NAN)).collect();\n"
478                );
479            }
480            None => body += &format!("        let f_{f} = g.f64({f:?})?;\n"),
481        }
482    }
483    for (i, (n, e)) in p.lines.iter().enumerate() {
484        body += &format!("        let v_{n}: Vec<f64> = {};\n", rs(e, &defined));
485        body += &format!("        cols[{i}].1.extend_from_slice(&v_{n});\n");
486        defined.insert(n.clone());
487    }
488    let names: Vec<String> = p.lines.iter().map(|(n, _)| format!("{n:?}")).collect();
489    Ok(format!(
490        r#"// Generated by {app} from a math code space. Do not edit.
491#![allow(unused, clippy::all)]
492use {sdk}::prelude::*;
493use {sdk}::series as s;
494
495const NAMES: &[&str] = &[{names}];
496
497pub fn run(host: &Host) -> Result<View> {{
498    let frame = host.input()?;
499{named}    let mut cols: Vec<(String, Vec<f64>)> = NAMES.iter().map(|n| (n.to_string(), Vec::with_capacity(frame.len()))).collect();
500    for grp in frame.groups() {{
501        let g = frame.slice(grp.start, grp.len);
502        let n = g.len();
503{body}    }}
504    {sdk}::columns::finish(host, &frame, &{out}, cols)
505}}
506"#,
507        names = names.join(", "),
508        named = named,
509        out = p.dirs.output_rs(profile.sdk_crate),
510        app = profile.app,
511        sdk = profile.sdk_crate,
512    ))
513}
514
515#[cfg(test)]
516mod tests {
517    use super::*;
518
519    #[test]
520    fn formulas_parse_with_precedence() {
521        let e = parse_expr("a + b * c ^ 2 ^ 1 > 3 and not d").unwrap();
522        let Expr::Bin("&&", l, r) = e else { panic!() };
523        assert!(matches!(*r, Expr::Not(_)));
524        let Expr::Bin(">", sum, _) = *l else { panic!() };
525        let Expr::Bin("+", _, prod) = *sum else { panic!() };
526        let Expr::Bin("*", _, pow) = *prod else { panic!() };
527        assert!(matches!(*pow, Expr::Bin("^", _, ref rhs) if matches!(**rhs, Expr::Bin("^", _, _))), "^ is right-associative");
528        assert_eq!(parse_expr("-x").unwrap(), Expr::Neg(Box::new(Expr::Name("x".into()))));
529        assert_eq!(parse_expr("1e3").unwrap(), Expr::Num(1000.0));
530        assert!(parse_expr("a +").is_err() && parse_expr("(a").is_err() && parse_expr("a b").is_err() && parse_expr("a $ b").is_err());
531    }
532
533    #[test]
534    fn programs_check_and_generate() {
535        let src = "# test\n@title T\n@view table\nfast = sma(close, 3)\nup = if(fast > close, 1, 0) # note\n@update signal = up\n@show close, fast";
536        let p = parse(src).unwrap();
537        assert_eq!(p.lines.len(), 2);
538        assert_eq!(p.fields().into_iter().collect::<Vec<_>>(), vec!["close".to_string()]);
539        assert_eq!((p.dirs.title.as_str(), p.dirs.view.as_str(), p.dirs.updates.len()), ("T", "table", 1));
540        let rs = generate(&crate::WEIRDNET, src).unwrap();
541        assert!(rs.contains("let f_close = g.f64(\"close\")?;") && rs.contains("s::sma(&f_close.clone(), &s::k(3.0, n))"), "{rs}");
542        let bad = |s: &str| parse(s).unwrap_err().to_string();
543        assert!(bad("x = nope(1)").contains("unknown function"));
544        assert!(bad("x = sma(close)").contains("takes 2"));
545        assert!(bad("\n\nx 1").contains("line 3"));
546        assert!(bad("sma = 1").contains("is a function"));
547        assert!(bad("@update a = b\nx = 1").contains("not one of the formulas"));
548        assert!(bad("@view pie\nx = 1").contains("@view"));
549        assert!(bad("# nothing").contains("no formulas"));
550    }
551
552    #[test]
553    fn named_inputs_are_read_by_dotted_names() {
554        let rs = generate(&crate::WEIRDNET, "spread = fast.close - slow.close\nboth = spread + close").unwrap();
555        assert!(rs.contains("let n_fast__close = host.input_named(\"fast\")?.f64(\"close\")?;"), "{rs}");
556        assert!(rs.contains("let f_slow__close: Vec<f64>") && rs.contains("let f_close = g.f64(\"close\")?;"), "{rs}");
557        assert!(rs.contains("s::zip(&f_fast__close.clone(), &f_slow__close.clone()"), "{rs}");
558        assert!(parse_expr("a.b.c").is_err() && parse_expr("a.").is_err());
559    }
560}