Skip to main content

weirdnet_code_spaces/
lib.rs

1//! # weirdnet-code-spaces
2//!
3//! Code spaces: small programs (math formulas, Polars expressions, Rust, typed Python, or a
4//! library built elsewhere) compiled into plugins against the `weirdnet-plugin` SDK, cached
5//! by content, loaded after an ABI check and approved by a trust hash.
6//!
7//! What stays with the application: where code spaces are stored, who may approve them,
8//! the host methods a plugin calls (its data) and how a result view is drawn ([`view`]
9//! parses, checks and converts it). Everything here is parameterized by a [`Profile`] (the SDK, symbol names, hash
10//! domains and the names messages use); [`WEIRDNET`] is the default.
11//!
12//! ```ignore
13//! use weirdnet_code_spaces::{self as cs, Env, Lang, WEIRDNET};
14//! # fn main() -> cs::Result<()> {
15//! let env = Env::new("/tmp/app/code", "/tmp/app/code/target");
16//! let src = "fast = sma(close, 10)";
17//! let built = cs::build(&WEIRDNET, &env, Lang::Math, src, &mut |line| eprintln!("{line}"))?;
18//! let plugin = cs::load(&WEIRDNET, &built.path)?; // then call plugin.run with a host table
19//! # Ok(()) }
20//! ```
21//!
22//! Features: `math` (default), `polars`, `python` (each adds its language; Python needs no
23//! libpython), `native-plugins` (default: running cargo and loading libraries, so desktop
24//! only). Images of results are not drawn here (see `weirdnet-render`).
25#![cfg_attr(docsrs, feature(doc_cfg))]
26
27/// The plugin C ABI (version 2 for [`WEIRDNET`]): the SDK's own `abi.rs`. Version 1 has the
28/// same layout, so a host table built from these types serves either.
29#[path = "../sdk/src/abi.rs"]
30pub mod abi;
31pub mod error;
32pub mod host;
33pub mod lang;
34#[cfg(feature = "math")]
35pub mod math;
36#[cfg(feature = "polars")]
37pub mod polars;
38#[cfg(feature = "python")]
39pub mod pyconv;
40#[cfg(feature = "python")]
41pub mod python;
42#[cfg(feature = "python")]
43#[allow(clippy::all, unused, unexpected_cfgs, deprecated)]
44mod python_ast;
45pub mod spaces;
46pub mod view;
47
48pub use error::{Result, SpacesError};
49pub use host::Host;
50pub use lang::Lang;
51
52use error::Error;
53use std::path::{Path, PathBuf};
54
55/// Everything that names the application or its SDK. Hashes, generated code, symbol names
56/// and messages all come from here, so an application keeps its formats by passing its own.
57#[derive(Debug, Clone, Copy)]
58pub struct Profile {
59    /// The application, in messages and generated-code comments ("not a Weirdnet plugin").
60    pub app: &'static str,
61    /// The SDK's package and crate names (`weirdnet-plugin`, `weirdnet_plugin`).
62    pub sdk_package: &'static str,
63    pub sdk_crate: &'static str,
64    pub sdk_version: &'static str,
65    /// The SDK crate's files (path, contents), written next to builds.
66    pub sdk_files: &'static [(&'static str, &'static str)],
67    /// The ABI version plugins must report.
68    pub abi_version: u32,
69    /// The three exported symbols (abi, info, run).
70    pub symbols: [&'static str; 3],
71    /// Domain separators of the trust and line-up hashes (end them with a NUL).
72    pub trust_domain: &'static [u8],
73    pub lineup_domain: &'static [u8],
74    /// Generated crates are named `{crate_prefix}{build key}`.
75    pub crate_prefix: &'static str,
76    /// Bumped when generated code changes (invalidates cached builds).
77    pub codegen: u32,
78    /// The variable that names cargo (else `cargo` on the PATH).
79    pub cargo_env: &'static str,
80    /// After "… compile to plugins with cargo (X not found). " when there is none.
81    pub cargo_hint: &'static str,
82    /// After "unknown function `f`" in math sources.
83    pub math_help: &'static str,
84}
85
86/// The `weirdnet-plugin` SDK this crate carries (`sdk/`, the same files as the
87/// `weirdnet-plugin` crate).
88pub const WEIRDNET_SDK_FILES: &[(&str, &str)] = &[
89    ("Cargo.toml", include_str!("../sdk/Cargo.toml.in")),
90    ("src/lib.rs", include_str!("../sdk/src/lib.rs")),
91    ("src/abi.rs", include_str!("../sdk/src/abi.rs")),
92    ("src/json.rs", include_str!("../sdk/src/json.rs")),
93    ("src/series.rs", include_str!("../sdk/src/series.rs")),
94    ("src/columns.rs", include_str!("../sdk/src/columns.rs")),
95    ("src/connector.rs", include_str!("../sdk/src/connector.rs")),
96    ("README.md", include_str!("../sdk/README.md")),
97    ("LICENSE", include_str!("../sdk/LICENSE")),
98];
99
100/// The default profile: the `weirdnet-plugin` SDK, ABI 2.
101pub const WEIRDNET: Profile = Profile {
102    app: "Weirdnet",
103    sdk_package: "weirdnet-plugin",
104    sdk_crate: "weirdnet_plugin",
105    sdk_version: "0.1.0",
106    sdk_files: WEIRDNET_SDK_FILES,
107    abi_version: abi::ABI_VERSION,
108    symbols: ["weirdnet_plugin_abi", "weirdnet_plugin_info", "weirdnet_plugin_run"],
109    trust_domain: b"weirdnet-code-trust\0",
110    lineup_domain: b"weirdnet-space-lineup\0",
111    crate_prefix: "wn_code_",
112    codegen: 1,
113    cargo_env: "WEIRDNET_CARGO",
114    cargo_hint: "Install Rust from https://rustup.rs, or set WEIRDNET_CARGO to cargo; prebuilt plugins run without it.",
115    math_help: "",
116};
117
118fn hex16(h: blake3::Hash) -> String {
119    h.to_hex()[..32].to_string()
120}
121
122/// Hash of the profile's SDK files.
123pub fn sdk_hash(p: &Profile) -> String {
124    let mut h = blake3::Hasher::new();
125    for (n, c) in p.sdk_files {
126        h.update(n.as_bytes());
127        h.update(&[0]);
128        h.update(c.as_bytes());
129        h.update(&[0]);
130    }
131    hex16(h.finalize())
132}
133
134/// Write the profile's SDK crate into `dir` (files already there and equal are left alone).
135pub fn write_sdk(p: &Profile, dir: &Path) -> Result<()> {
136    for (n, c) in p.sdk_files {
137        let f = dir.join(n);
138        if let Some(parent) = f.parent() {
139            std::fs::create_dir_all(parent)?;
140        }
141        if std::fs::read_to_string(&f).ok().as_deref() != Some(*c) {
142            std::fs::write(&f, c)?;
143        }
144    }
145    Ok(())
146}
147
148/// Where code spaces are built and cached. The application chooses both directories.
149#[derive(Debug, Clone, PartialEq)]
150pub struct Env {
151    /// The code directory: `sdk/`, `build/` and `plugins/` go under it.
152    pub root: PathBuf,
153    /// Cargo's target directory (shared by every build).
154    pub target: PathBuf,
155}
156
157impl Env {
158    pub fn new(root: impl Into<PathBuf>, target: impl Into<PathBuf>) -> Self {
159        Env { root: root.into(), target: target.into() }
160    }
161    pub fn sdk_dir(&self, p: &Profile) -> PathBuf {
162        self.root.join("sdk").join(sdk_dir_name(p))
163    }
164    pub fn build_dir(&self, key: &str) -> PathBuf {
165        self.root.join("build").join(key)
166    }
167    pub fn plugins_dir(&self) -> PathBuf {
168        self.root.join("plugins")
169    }
170    /// The cached library of a build key.
171    pub fn plugin_path(&self, key: &str) -> PathBuf {
172        self.plugins_dir().join(format!("{key}.{}", std::env::consts::DLL_EXTENSION))
173    }
174}
175
176fn sdk_dir_name(p: &Profile) -> String {
177    format!("{}-{}", p.sdk_package, &sdk_hash(p)[..12])
178}
179
180/// The library a `plugin` source names (its path; `~/` is the home directory).
181pub fn plugin_file(source: &str) -> PathBuf {
182    let s = source.trim();
183    match s.strip_prefix("~/") {
184        Some(rest) => dirs::home_dir().map(|h| h.join(rest)).unwrap_or_else(|| PathBuf::from(s)),
185        None => PathBuf::from(s),
186    }
187}
188
189/// What an owner approves: the language and source (or the library's bytes), under the
190/// profile's trust domain.
191pub fn trust_hash_of(p: &Profile, lang: Lang, source: &str) -> Result<String> {
192    let mut h = blake3::Hasher::new();
193    h.update(p.trust_domain);
194    h.update(lang.as_str().as_bytes());
195    h.update(&[0]);
196    if lang == Lang::Plugin {
197        let f = plugin_file(source);
198        let bytes = std::fs::read(&f).map_err(|e| Error::invalid("plugin", format!("{}: {e}", f.display())))?;
199        h.update(&bytes);
200    } else {
201        h.update(source.as_bytes());
202    }
203    Ok(hex16(h.finalize()))
204}
205
206/// Approved as it is: `approved_hash` is the trust hash of this language and source (a
207/// library that can't be read is never approved).
208pub fn approved(p: &Profile, lang: Lang, source: &str, approved_hash: Option<&str>) -> bool {
209    trust_hash_of(p, lang, source).ok().is_some_and(|h| approved_hash == Some(h.as_str()))
210}
211
212/// The extra dependency line a language's generated crate needs.
213pub fn dependency(lang: Lang) -> &'static str {
214    match lang {
215        #[cfg(feature = "polars")]
216        Lang::Polars => polars::DEPENDENCY,
217        #[cfg(feature = "python")]
218        Lang::Python => python::DEPENDENCY,
219        _ => "",
220    }
221}
222
223/// The cache key of a build: ABI, SDK, codegen, language, platform, dependencies, source.
224pub fn build_key(p: &Profile, lang: Lang, source: &str) -> String {
225    let mut h = blake3::Hasher::new();
226    h.update(
227        format!(
228            "abi{}|sdk{}|codegen{}|{}|{}-{}\0",
229            p.abi_version,
230            sdk_hash(p),
231            p.codegen,
232            lang.as_str(),
233            std::env::consts::ARCH,
234            std::env::consts::OS
235        )
236        .as_bytes(),
237    );
238    h.update(dependency(lang).as_bytes());
239    h.update(&[0]);
240    h.update(source.as_bytes());
241    hex16(h.finalize())
242}
243
244fn unsupported(lang: Lang) -> Error {
245    Error::invalid("code", format!("{} code spaces need this crate's `{}` feature", lang.as_str(), lang.as_str()))
246}
247
248/// The cargo to build with: the profile's variable or `cargo` on the PATH, checked by
249/// running it.
250pub fn cargo(p: &Profile) -> Result<PathBuf> {
251    let c = std::env::var_os(p.cargo_env).filter(|c| !c.is_empty()).map(PathBuf::from).unwrap_or_else(|| PathBuf::from("cargo"));
252    match std::process::Command::new(&c).arg("--version").output() {
253        Ok(o) if o.status.success() => Ok(c),
254        _ => Err(Error::missing(
255            "Rust toolchain",
256            format!("code spaces in math, polars, rust and python compile to plugins with cargo ({} not found). {}", c.display(), p.cargo_hint),
257        )),
258    }
259}
260
261/// Check a source without building: the language's own checks (a prebuilt plugin is
262/// copied into the cache and loaded, never loaded in place).
263pub fn check(p: &Profile, env: &Env, lang: Lang, source: &str) -> Result<Vec<String>> {
264    let _ = env;
265    Ok(match lang {
266        #[cfg(feature = "math")]
267        Lang::Math => {
268            let m = math::parse_for(p, source)?;
269            let fields: Vec<String> = m.fields().into_iter().collect();
270            vec![format!("{} formula(s); input fields used: {}", m.lines.len(), if fields.is_empty() { "none".into() } else { fields.join(", ") })]
271        }
272        #[cfg(feature = "polars")]
273        Lang::Polars => {
274            let m = polars::parse(source)?;
275            vec![format!("{} step(s); expressions are checked by the compiler at build time", m.steps.len())]
276        }
277        #[cfg(feature = "python")]
278        Lang::Python => {
279            let m = python::parse(source)?;
280            let public: Vec<&str> = m.funcs.iter().filter(|f| !f.name.starts_with('_')).map(|f| f.name.as_str()).collect();
281            pyconv::to_rust(source, "user.py").map_err(|e| Error::invalid("python", e))?;
282            vec![format!("columns: {}", public.join(", "))]
283        }
284        Lang::Rust => {
285            if !source.contains("fn run") {
286                return Err(Error::invalid(
287                    "rust",
288                    format!("define `pub fn run(host: &Host) -> Result<View>` (with `use {}::prelude::*;`)", p.sdk_crate),
289                ));
290            }
291            vec!["defines run; the compiler checks the rest at build time".into()]
292        }
293        #[cfg(feature = "native-plugins")]
294        Lang::Plugin => {
295            let f = plugin_file(source);
296            let l = load(p, &build(p, env, lang, source, &mut |_| {})?.path)?;
297            vec![format!("{}: ABI {} ({})", f.display(), p.abi_version, l.info)]
298        }
299        #[allow(unreachable_patterns)]
300        other => return Err(unsupported(other)),
301    })
302}
303
304/// A built plugin.
305#[derive(Debug, Clone, PartialEq)]
306pub struct Built {
307    pub path: PathBuf,
308    pub key: String,
309    /// Already built (nothing compiled).
310    pub cached: bool,
311}
312
313/// The `Cargo.toml` of a generated crate.
314pub fn manifest(p: &Profile, key: &str, sdk: &Path, extra: &str) -> String {
315    format!(
316        "[package]\nname = \"{}{key}\"\nversion = \"0.1.0\"\nedition = \"2021\"\npublish = false\n\n[lib]\ncrate-type = [\"cdylib\"]\npath = \"src/lib.rs\"\n\n[dependencies]\n{} = {{ path = {:?} }}\n{extra}\n\n[profile.release]\nopt-level = 3\ndebug = false\n\n[workspace]\n",
317        p.crate_prefix,
318        p.sdk_package,
319        sdk.display().to_string()
320    )
321}
322
323/// The generated crate's `user.rs` (and `py.rs` for Python) for a source.
324pub fn generate_user(p: &Profile, lang: Lang, source: &str) -> Result<(String, Option<String>, &'static str)> {
325    let _ = p;
326    Ok(match lang {
327        #[cfg(feature = "math")]
328        Lang::Math => (math::generate(p, source)?, None, ""),
329        #[cfg(feature = "polars")]
330        Lang::Polars => (polars::generate(p, source)?, None, polars::DEPENDENCY),
331        Lang::Rust => {
332            if !source.contains("fn run") {
333                return Err(Error::invalid("rust", "define `pub fn run(host: &Host) -> Result<View>`"));
334            }
335            (source.to_string(), None, "")
336        }
337        #[cfg(feature = "python")]
338        Lang::Python => {
339            let user = python::generate(p, source)?;
340            let py = python::transpile(source, Path::new("."))?;
341            (user, Some(py), python::DEPENDENCY)
342        }
343        Lang::Plugin => return Err(Error::invalid("plugin", "prebuilt plugins are not compiled")),
344        #[allow(unreachable_patterns)]
345        other => return Err(unsupported(other)),
346    })
347}
348
349/// The generated crate's `lib.rs`.
350pub fn lib_rs(p: &Profile, lang: Lang) -> String {
351    format!("#![allow(unused, clippy::all)]\nmod user;\n{}::plugin!(user::run, {:?});\n", p.sdk_crate, lang.as_str())
352}
353
354/// Write the crate of a source into `dir` (a build directory under the code root).
355pub fn generate(p: &Profile, lang: Lang, source: &str, dir: &Path) -> Result<()> {
356    let src = dir.join("src");
357    std::fs::create_dir_all(&src)?;
358    let (user, py, extra) = generate_user(p, lang, source)?;
359    if let Some(py) = py {
360        std::fs::write(src.join("py.rs"), py)?;
361    }
362    std::fs::write(src.join("user.rs"), user)?;
363    std::fs::write(src.join("lib.rs"), lib_rs(p, lang))?;
364    let key = dir.file_name().and_then(|n| n.to_str()).unwrap_or("x");
365    std::fs::write(dir.join("Cargo.toml"), manifest(p, key, &dir.join("../../sdk").join(sdk_dir_name(p)), extra))?;
366    Ok(())
367}
368
369#[cfg(feature = "native-plugins")]
370mod native;
371#[cfg(feature = "native-plugins")]
372pub use native::{Loaded, build, load, run};
373
374#[cfg(test)]
375mod tests;