weirdnet-code-spaces 0.1.0

Weirdnet code spaces: math, Polars, Rust and typed Python sources compiled into plugins against the weirdnet-plugin SDK, content-addressed builds, ABI-checked loading, trust hashes and space line-ups, under an application profile.
Documentation
//! Python code spaces, translated to Rust in process (ruff's parser and python-ast's code
//! generation, no libpython) and then compiled to a plugin.
//!
//! ```text
//! # @title Range
//! # @update spread = spread
//! def _half(x: float) -> float:
//!     return x / 2.0
//!
//! def spread(high: float, low: float) -> float:
//!     return _half(high - low) * 2.0
//! ```
//!
//! Every top-level public `def` is a column: it runs once per record, its parameters are
//! the record's fields (by name) and it returns `float`, `int` or `bool`. Parameters are
//! `float`, `int`, `bool` or `str`. `_private` functions are helpers. Directives are those
//! of math code spaces, written as `# @…` comments.
use super::math::{Directives, ident};
use crate::Profile;
use crate::error::{Error, Result};

/// The stdpython crate the generated Rust needs (no libpython: the `alloc` runtime only).
pub const DEPENDENCY: &str = r#"stdpython = { version = "1.1", default-features = false, features = ["alloc"] }"#;

#[derive(Debug, Clone, PartialEq)]
pub struct Func {
    pub name: String,
    pub params: Vec<(String, String)>,
    pub ret: String,
}

#[derive(Debug, Clone, Default, PartialEq)]
pub struct Program {
    pub funcs: Vec<Func>,
    pub dirs: Directives,
}

const PARAM_TYPES: &[&str] = &["float", "int", "bool", "str"];
const RETURN_TYPES: &[&str] = &["float", "int", "bool"];

fn signature(l: &str) -> std::result::Result<Func, String> {
    let rest = l.strip_prefix("def ").unwrap_or(l).trim();
    let (name, rest) = rest.split_once('(').ok_or("def NAME(…) -> TYPE:")?;
    let name = name.trim();
    ident(name)?;
    let (params, rest) = rest.rsplit_once(')').ok_or("def NAME(…) -> TYPE:")?;
    let ret = rest.trim().strip_suffix(':').and_then(|r| r.trim().strip_prefix("->")).map(str::trim).unwrap_or("");
    let public = !name.starts_with('_');
    if public && !RETURN_TYPES.contains(&ret) {
        return Err(format!("def {name}: annotate the return type (-> float, int or bool)"));
    }
    let mut ps = vec![];
    for p in params.split(',').map(str::trim).filter(|p| !p.is_empty()) {
        let (n, t) = p.split_once(':').map(|(n, t)| (n.trim(), t.trim())).unwrap_or((p, ""));
        let t = t.split_once('=').map(|(t, _)| t.trim()).unwrap_or(t);
        ident(n)?;
        if t.is_empty() {
            return Err(format!("def {name}: annotate parameter `{n}` (Python code spaces need types)"));
        }
        if public && !PARAM_TYPES.contains(&t) {
            return Err(format!("def {name}: parameter `{n}: {t}` must be float, int, bool or str"));
        }
        ps.push((n.to_string(), t.to_string()));
    }
    Ok(Func { name: name.to_string(), params: ps, ret: ret.to_string() })
}

/// Read the functions and directives of a Python source.
pub fn parse(src: &str) -> Result<Program> {
    let mut p = Program::default();
    let mut lines = src.lines().enumerate().peekable();
    while let Some((i, raw)) = lines.next() {
        let err = |m: String| Error::invalid("python", format!("line {}: {m}", i + 1));
        let t = raw.trim_end();
        if let Some(d) = t.trim_start().strip_prefix('#').map(str::trim).and_then(|c| c.strip_prefix('@')) {
            p.dirs.read(d).map_err(err)?;
            continue;
        }
        if t.starts_with("def ") {
            // a signature may wrap over lines until the `:`
            let mut sig = t.to_string();
            while !sig.ends_with(':') {
                match lines.next() {
                    Some((_, more)) => sig.push_str(more.trim()),
                    None => return Err(err("the def line does not end with `:`".into())),
                }
            }
            let f = signature(&sig).map_err(err)?;
            if p.funcs.iter().any(|g| g.name == f.name) {
                return Err(err(format!("`{}` is defined twice", f.name)));
            }
            p.funcs.push(f);
        } else if t.starts_with("class ") || t.starts_with("async ") {
            return Err(err("only functions are supported at the top level".into()));
        }
    }
    if !p.funcs.iter().any(|f| !f.name.starts_with('_')) {
        return Err(Error::invalid("python", "no public functions: write `def name(field: float) -> float:`"));
    }
    for (_, c) in &p.dirs.updates {
        if !p.funcs.iter().any(|f| &f.name == c && !c.starts_with('_')) {
            return Err(Error::invalid("python", format!("@update: `{c}` is not one of the public functions")));
        }
    }
    Ok(p)
}

/// Translate Python to Rust (`py.rs`) in process: parsed by ruff, generated by python-ast.
pub fn transpile(src: &str, _dir: &std::path::Path) -> Result<String> {
    super::pyconv::to_rust(src, "user.py").map_err(|e| Error::invalid("python", e))
}

/// The Rust `user.rs` wrapper calling the translated Python (`py.rs`) once per record.
pub fn generate(profile: &Profile, src: &str) -> Result<String> {
    let p = parse(src)?;
    let public: Vec<&Func> = p.funcs.iter().filter(|f| !f.name.starts_with('_')).collect();
    let mut body = String::new();
    for (i, f) in public.iter().enumerate() {
        let mut args = vec![];
        let mut ok = vec![];
        for (n, t) in &f.params {
            let v = format!("frame.get(r, {n:?})");
            args.push(match t.as_str() {
                "float" => format!("{v}.as_f64().unwrap_or(f64::NAN)"),
                "int" => {
                    ok.push(format!("{v}.as_f64().is_some()"));
                    format!("{v}.as_f64().map(|x| x as i64).unwrap_or(0)")
                }
                "bool" => format!("{v}.as_bool().or_else(|| {v}.as_f64().map(|x| x != 0.0)).unwrap_or(false)"),
                _ => format!("{v}.text()"),
            });
        }
        let call = format!("py::{}({})", f.name, args.join(", "));
        let conv = match f.ret.as_str() {
            "bool" => "if v { 1.0 } else { 0.0 }",
            "int" => "v as f64",
            _ => "v",
        };
        let guard = if ok.is_empty() { "true".to_string() } else { ok.join(" && ") };
        body += &format!(
            "        cols[{i}].1.push(if {guard} {{ match {call} {{ Ok(v) => {conv}, Err(e) => return Err(Error::new(format!(\"{name}: {{e}} (record {{}})\", frame.ids[r]))) }} }} else {{ f64::NAN }});\n",
            name = f.name
        );
    }
    for f in &public {
        for (n, _) in &f.params {
            body = format!("    if !frame.has({n:?}) {{ return Err(Error::new(\"{}: the input has no field `{n}`\")); }}\n", f.name) + &body;
        }
    }
    let names: Vec<String> = public.iter().map(|f| format!("{:?}", f.name)).collect();
    Ok(format!(
        r#"// Generated by {app} from a Python code space (via python-ast). Do not edit.
#![allow(unused, clippy::all)]
use {sdk}::prelude::*;
#[path = "py.rs"]
mod py;

const NAMES: &[&str] = &[{names}];

pub fn run(host: &Host) -> Result<View> {{
    let frame = host.input()?;
    let mut cols: Vec<(String, Vec<f64>)> = NAMES.iter().map(|n| (n.to_string(), Vec::with_capacity(frame.len()))).collect();
    for r in 0..frame.len() {{
{body}    }}
    {sdk}::columns::finish(host, &frame, &{dirs}, cols)
}}
"#,
        names = names.join(", "),
        dirs = p.dirs.output_rs(profile.sdk_crate),
        app = profile.app,
        sdk = profile.sdk_crate,
    ))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn python_signatures_are_checked() {
        let src = "# @title T\n# @update s = spread\nimport math\ndef _h(x: float) -> float:\n    return x\n\ndef spread(high: float,\n           low: float, n: int) -> float:\n    return high - low\n\ndef up(close: float) -> bool:\n    return close > 1.0\n";
        let p = parse(src).unwrap();
        assert_eq!(p.funcs.len(), 3);
        assert_eq!(p.funcs[1].params, vec![("high".into(), "float".into()), ("low".into(), "float".into()), ("n".into(), "int".into())]);
        assert_eq!(p.dirs.title, "T");
        let rs = generate(&crate::WEIRDNET, src).unwrap();
        assert!(rs.contains("py::spread(frame.get(r, \"high\")"), "{rs}");
        assert!(rs.contains("py::up("));
        assert!(!rs.contains("py::_h"));
        assert!(parse("def f(x) -> float:\n  return x").unwrap_err().to_string().contains("annotate parameter"));
        assert!(parse("def f(x: float):\n  return x").unwrap_err().to_string().contains("return type"));
        assert!(parse("def f(x: list) -> float:\n  return 1.0").unwrap_err().to_string().contains("must be float"));
        assert!(parse("def _f(x: float) -> float:\n  return x").unwrap_err().to_string().contains("no public"));
        assert!(parse("class A:\n  pass\ndef f(x: float) -> float:\n  return x").unwrap_err().to_string().contains("only functions"));
    }
}