Skip to main content

graphrt_bundle/
builder.rs

1//! Building bundles.
2
3use std::collections::{BTreeMap, BTreeSet};
4use std::path::Path;
5
6use graphrt_core::Workflow;
7
8use crate::archive::{validate_path, write_zip};
9use crate::error::BundleError;
10use crate::manifest::{BUNDLE_FORMAT_VERSION, BundleManifest, MANIFEST_PATH, PluginRef, Requires, SIGNATURE_PATH};
11use crate::secrets::check_no_inline_secrets;
12use crate::sha256_hex;
13
14/// Directories never copied from plugin or asset folders.
15const SKIP_DIRS: &[&str] = &[".git", "target", "node_modules", "__pycache__", ".venv"];
16/// Largest single file accepted by the builder.
17const MAX_FILE_BYTES: u64 = 64 * 1024 * 1024;
18
19/// Assembles a bundle.
20#[derive(Clone, Debug, Default)]
21pub struct BundleBuilder {
22    name: String,
23    version: String,
24    description: String,
25    created_at: Option<String>,
26    workflows: Vec<String>,
27    files: BTreeMap<String, Vec<u8>>,
28    plugins: Vec<PluginRef>,
29}
30
31impl BundleBuilder {
32    /// New bundle named `name` at `version`.
33    pub fn new(name: impl Into<String>, version: impl Into<String>) -> Self {
34        Self { name: name.into(), version: version.into(), ..Self::default() }
35    }
36
37    pub fn description(mut self, description: impl Into<String>) -> Self {
38        self.description = description.into();
39        self
40    }
41
42    /// Fixed creation time (RFC 3339) for reproducible builds. Defaults to
43    /// `$SOURCE_DATE_EPOCH` when set, else the current time.
44    pub fn created_at(mut self, created_at: impl Into<String>) -> Self {
45        self.created_at = Some(created_at.into());
46        self
47    }
48
49    /// Add a workflow document stored at `path` (e.g. `workflow.yaml`). The first
50    /// workflow added is the entry. The document must parse.
51    pub fn workflow(mut self, path: &str, text: impl Into<String>) -> Result<Self, BundleError> {
52        let text = text.into();
53        validate_path(path)?;
54        reserved(path)?;
55        Workflow::parse(&text).map_err(|e| BundleError::Workflow { name: path.to_owned(), message: e.to_string() })?;
56        if self.workflows.iter().any(|w| w == path) {
57            return Err(BundleError::Build(format!("workflow `{path}` added twice")));
58        }
59        self.workflows.push(path.to_owned());
60        self.files.insert(path.to_owned(), text.into_bytes());
61        Ok(self)
62    }
63
64    /// Add an arbitrary file (assets, data) at `path`.
65    pub fn file(mut self, path: &str, bytes: Vec<u8>) -> Result<Self, BundleError> {
66        validate_path(path)?;
67        reserved(path)?;
68        if self.files.insert(path.to_owned(), bytes).is_some() {
69            return Err(BundleError::Build(format!("`{path}` added twice")));
70        }
71        Ok(self)
72    }
73
74    /// Copy a directory under `prefix/` (e.g. `assets`), skipping build output and VCS folders.
75    pub fn include_dir(mut self, dir: &Path, prefix: &str) -> Result<Self, BundleError> {
76        for (rel, bytes) in read_tree(dir)? {
77            self = self.file(&format!("{prefix}/{rel}"), bytes)?;
78        }
79        Ok(self)
80    }
81
82    /// Add a plugin directory (with `graphrt-plugin.json` or `.yaml`) under
83    /// `plugins/<name>/`. The manifest is stored as JSON. WebAssembly plugins are
84    /// portable; process and http plugins are kept but flagged non-portable.
85    pub fn plugin_dir(mut self, dir: &Path) -> Result<Self, BundleError> {
86        let manifest = read_plugin_manifest(dir)?;
87        let name = manifest
88            .get("name")
89            .and_then(|v| v.as_str())
90            .filter(|n| !n.is_empty() && !n.contains('/') && !n.contains(".."))
91            .ok_or_else(|| BundleError::Build(format!("{}: plugin manifest needs a simple `name`", dir.display())))?
92            .to_owned();
93        let version = manifest.get("version").and_then(|v| v.as_str()).unwrap_or_default().to_owned();
94        let runtime = manifest.get("runtime").and_then(|v| v.as_str()).unwrap_or("process").to_owned();
95        if self.plugins.iter().any(|p| p.name == name) {
96            return Err(BundleError::Build(format!("plugin `{name}` added twice")));
97        }
98        check_no_inline_secrets(&manifest, &format!("plugin `{name}` manifest"))?;
99        let manifest_bytes = serde_json::to_vec_pretty(&manifest).map_err(|e| BundleError::Build(e.to_string()))?;
100        let prefix = format!("plugins/{name}");
101        for (rel, bytes) in read_tree(dir)? {
102            if rel.starts_with("graphrt-plugin.") {
103                continue;
104            }
105            self = self.file(&format!("{prefix}/{rel}"), bytes)?;
106        }
107        if runtime == "wasm" {
108            let module = manifest.get("module").and_then(|v| v.as_str()).unwrap_or_default();
109            if !self.files.contains_key(&format!("{prefix}/{module}")) {
110                return Err(BundleError::Build(format!("plugin `{name}`: module `{module}` not found")));
111            }
112        }
113        self.plugins.push(PluginRef {
114            name: name.clone(),
115            version,
116            portable: runtime == "wasm",
117            runtime,
118            sha256: sha256_hex(&manifest_bytes),
119        });
120        self.files.insert(format!("{prefix}/graphrt-plugin.json"), manifest_bytes);
121        Ok(self)
122    }
123
124    /// Add a bare, self-describing WebAssembly module as plugin `name` (a manifest is
125    /// generated; its node kinds may be plain names).
126    pub fn wasm_module(mut self, name: &str, module: Vec<u8>) -> Result<Self, BundleError> {
127        if name.is_empty() || name.contains('/') || name.contains("..") {
128            return Err(BundleError::Build(format!("invalid plugin name `{name}`")));
129        }
130        if self.plugins.iter().any(|p| p.name == name) {
131            return Err(BundleError::Build(format!("plugin `{name}` added twice")));
132        }
133        let manifest = serde_json::json!({
134            "name": name,
135            "version": "",
136            "runtime": "wasm",
137            "module": "module.wasm",
138            "allow_unprefixed_kinds": true,
139        });
140        let manifest_bytes = serde_json::to_vec_pretty(&manifest).map_err(|e| BundleError::Build(e.to_string()))?;
141        self = self.file(&format!("plugins/{name}/module.wasm"), module)?;
142        self.plugins.push(PluginRef {
143            name: name.to_owned(),
144            version: String::new(),
145            runtime: "wasm".into(),
146            portable: true,
147            sha256: sha256_hex(&manifest_bytes),
148        });
149        self.files.insert(format!("plugins/{name}/graphrt-plugin.json"), manifest_bytes);
150        Ok(self)
151    }
152
153    /// Add a model providers file (YAML or JSON). Files with inline secrets are rejected.
154    pub fn providers(mut self, text: &str) -> Result<Self, BundleError> {
155        let value = parse_structured(text, "providers file")?;
156        check_no_inline_secrets(&value, "providers file")?;
157        let bytes = serde_json::to_vec_pretty(&value).map_err(|e| BundleError::Build(e.to_string()))?;
158        self.files.insert("providers.json".into(), bytes);
159        Ok(self)
160    }
161
162    /// Add a knowledge-base config (YAML or JSON). Stores whose `path` exists under
163    /// `base_dir` (memory JSON files, SQLite databases) are copied into
164    /// `knowledge/` and the path rewritten, so a prebuilt index ships with the
165    /// bundle. Inline secrets are rejected, and so is a SQLite database whose
166    /// write-ahead log still holds commits (the copy would silently miss them).
167    pub fn knowledge(mut self, text: &str, base_dir: &Path) -> Result<Self, BundleError> {
168        let mut value = parse_structured(text, "knowledge config")?;
169        check_no_inline_secrets(&value, "knowledge config")?;
170        if let Some(stores) = value.get_mut("stores").and_then(|s| s.as_object_mut()) {
171            for (store_name, store) in stores.iter_mut() {
172                let Some(path) = store.get("path").and_then(|p| p.as_str()).map(str::to_owned) else { continue };
173                let source = base_dir.join(&path);
174                if !source.is_file() {
175                    continue;
176                }
177                reject_uncheckpointed_sqlite(store_name, &source)?;
178                let bytes = read_file(&source)?;
179                let file_name = source.file_name().map(|f| f.to_string_lossy().into_owned()).unwrap_or_default();
180                let file_name = file_name.trim_start_matches('.');
181                let target = format!("knowledge/{store_name}-{file_name}");
182                if let Some(obj) = store.as_object_mut() {
183                    obj.insert("path".into(), serde_json::Value::String(target.clone()));
184                }
185                self = self.file(&target, bytes)?;
186            }
187        }
188        let bytes = serde_json::to_vec_pretty(&value).map_err(|e| BundleError::Build(e.to_string()))?;
189        self.files.insert("knowledge/knowledge.json".into(), bytes);
190        Ok(self)
191    }
192
193    /// Finish: compute requirements and hashes and produce the archive bytes.
194    pub fn build(self) -> Result<Vec<u8>, BundleError> {
195        let entry = self.workflows.first().cloned().ok_or_else(|| BundleError::Build("no workflow added".into()))?;
196        let mut node_kinds = BTreeSet::new();
197        let mut providers = BTreeSet::new();
198        let mut env = BTreeSet::new();
199        for path in &self.workflows {
200            let text = self
201                .files
202                .get(path)
203                .and_then(|b| std::str::from_utf8(b).ok())
204                .ok_or_else(|| BundleError::Build(format!("workflow `{path}` is not UTF-8")))?;
205            let wf = Workflow::parse(text)
206                .map_err(|e| BundleError::Workflow { name: path.clone(), message: e.to_string() })?;
207            for decl in &wf.env {
208                env.insert(decl.name.clone());
209            }
210            for node in &wf.graph.nodes {
211                node_kinds.insert(node.kind.clone());
212                if let Some(provider) =
213                    node.config.get("model").and_then(|m| m.get("provider")).and_then(|p| p.as_str())
214                {
215                    providers.insert(provider.to_owned());
216                }
217            }
218        }
219        let files: BTreeMap<String, String> = self.files.iter().map(|(p, b)| (p.clone(), sha256_hex(b))).collect();
220        let manifest = BundleManifest {
221            format_version: BUNDLE_FORMAT_VERSION,
222            name: self.name,
223            version: self.version,
224            description: self.description,
225            created_at: self.created_at.unwrap_or_else(now_rfc3339),
226            entry,
227            workflows: self.workflows,
228            requires: Requires {
229                graphrt: runtime_requirement(),
230                node_kinds: node_kinds.into_iter().collect(),
231                plugins: self.plugins,
232                providers: providers.into_iter().collect(),
233                env: env.into_iter().collect(),
234            },
235            files,
236        };
237        if manifest.name.trim().is_empty() {
238            return Err(BundleError::Build("bundle `name` must not be empty".into()));
239        }
240        let manifest_bytes = serde_json::to_vec_pretty(&manifest).map_err(|e| BundleError::Build(e.to_string()))?;
241        write_zip(&[(MANIFEST_PATH, &manifest_bytes)], &self.files)
242    }
243}
244
245fn reserved(path: &str) -> Result<(), BundleError> {
246    if path == MANIFEST_PATH || path == SIGNATURE_PATH {
247        return Err(BundleError::Build(format!("`{path}` is reserved")));
248    }
249    Ok(())
250}
251
252/// `>=major.minor` of this runtime.
253fn runtime_requirement() -> String {
254    let version = env!("CARGO_PKG_VERSION");
255    let mut parts = version.split('.');
256    match (parts.next(), parts.next()) {
257        (Some(major), Some(minor)) => format!(">={major}.{minor}"),
258        _ => String::new(),
259    }
260}
261
262fn parse_structured(text: &str, what: &str) -> Result<serde_json::Value, BundleError> {
263    if text.trim_start().starts_with(['{', '[']) {
264        return serde_json::from_str(text).map_err(|e| BundleError::Build(format!("invalid {what}: {e}")));
265    }
266    let options = serde_saphyr::options! { strict_booleans: true };
267    serde_saphyr::from_str_with_options(text, options).map_err(|e| BundleError::Build(format!("invalid {what}: {e}")))
268}
269
270fn read_plugin_manifest(dir: &Path) -> Result<serde_json::Value, BundleError> {
271    for name in ["graphrt-plugin.json", "graphrt-plugin.yaml", "graphrt-plugin.yml"] {
272        let path = dir.join(name);
273        if path.is_file() {
274            let bytes = read_file(&path)?;
275            let text =
276                String::from_utf8(bytes).map_err(|_| BundleError::Build(format!("{}: not UTF-8", path.display())))?;
277            return parse_structured(&text, &format!("plugin manifest {}", path.display()));
278        }
279    }
280    Err(BundleError::Build(format!("{}: no graphrt-plugin.json or graphrt-plugin.yaml", dir.display())))
281}
282
283fn read_file(path: &Path) -> Result<Vec<u8>, BundleError> {
284    let meta = std::fs::symlink_metadata(path).map_err(|e| BundleError::io(path, &e))?;
285    if !meta.is_file() {
286        return Err(BundleError::Build(format!("{}: not a regular file (symlinks are not bundled)", path.display())));
287    }
288    if meta.len() > MAX_FILE_BYTES {
289        return Err(BundleError::Build(format!("{}: larger than {MAX_FILE_BYTES} bytes", path.display())));
290    }
291    std::fs::read(path).map_err(|e| BundleError::io(path, &e))
292}
293
294/// Fail when `source` is a SQLite database with a non-empty `-wal` file beside
295/// it: those commits are not in the main file yet (a writer is still open, or
296/// one crashed). Closing every connection checkpoints and removes the log.
297fn reject_uncheckpointed_sqlite(store_name: &str, source: &Path) -> Result<(), BundleError> {
298    let mut wal_name = source.as_os_str().to_owned();
299    wal_name.push("-wal");
300    let wal = Path::new(&wal_name);
301    let pending = std::fs::metadata(wal).is_ok_and(|m| m.is_file() && m.len() > 0);
302    if !pending {
303        return Ok(());
304    }
305    Err(BundleError::Build(format!(
306        "store `{store_name}`: {} has uncommitted write-ahead log {}; stop every process using the \
307         database (or run `sqlite3 {} 'PRAGMA wal_checkpoint(TRUNCATE)'`) and build again",
308        source.display(),
309        wal.display(),
310        source.display()
311    )))
312}
313
314/// Regular files under `dir` as (`/`-separated relative path, bytes), sorted.
315/// Symlinks and build / VCS folders are skipped.
316fn read_tree(dir: &Path) -> Result<Vec<(String, Vec<u8>)>, BundleError> {
317    let mut out = Vec::new();
318    let mut stack = vec![(dir.to_path_buf(), String::new())];
319    while let Some((path, rel)) = stack.pop() {
320        let entries = std::fs::read_dir(&path).map_err(|e| BundleError::io(&path, &e))?;
321        for entry in entries {
322            let entry = entry.map_err(|e| BundleError::io(&path, &e))?;
323            let name = entry.file_name().to_string_lossy().into_owned();
324            let child_rel = if rel.is_empty() { name.clone() } else { format!("{rel}/{name}") };
325            let file_type = entry.file_type().map_err(|e| BundleError::io(&entry.path(), &e))?;
326            if file_type.is_symlink() {
327                continue;
328            }
329            if file_type.is_dir() {
330                if !SKIP_DIRS.contains(&name.as_str()) {
331                    stack.push((entry.path(), child_rel));
332                }
333            } else if file_type.is_file() {
334                out.push((child_rel, read_file(&entry.path())?));
335            }
336        }
337    }
338    out.sort_by(|a, b| a.0.cmp(&b.0));
339    Ok(out)
340}
341
342/// Current time as RFC 3339 UTC, or `$SOURCE_DATE_EPOCH` for reproducible builds.
343fn now_rfc3339() -> String {
344    let secs = std::env::var("SOURCE_DATE_EPOCH").ok().and_then(|s| s.parse::<i64>().ok()).unwrap_or_else(|| {
345        std::time::SystemTime::now()
346            .duration_since(std::time::UNIX_EPOCH)
347            .map(|d| i64::try_from(d.as_secs()).unwrap_or(0))
348            .unwrap_or(0)
349    });
350    rfc3339(secs)
351}
352
353/// Format Unix seconds as `YYYY-MM-DDTHH:MM:SSZ` (proleptic Gregorian calendar).
354pub(crate) fn rfc3339(secs: i64) -> String {
355    let days = secs.div_euclid(86_400);
356    let rem = secs.rem_euclid(86_400);
357    // Civil-from-days (Howard Hinnant's algorithm).
358    let z = days + 719_468;
359    let era = z.div_euclid(146_097);
360    let doe = z.rem_euclid(146_097);
361    let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
362    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
363    let mp = (5 * doy + 2) / 153;
364    let day = doy - (153 * mp + 2) / 5 + 1;
365    let month = if mp < 10 { mp + 3 } else { mp - 9 };
366    let year = yoe + era * 400 + i64::from(month <= 2);
367    format!("{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z", rem / 3_600, (rem % 3_600) / 60, rem % 60)
368}
369
370#[cfg(test)]
371mod tests {
372    use super::*;
373
374    #[test]
375    fn rfc3339_formatting() {
376        assert_eq!(rfc3339(0), "1970-01-01T00:00:00Z");
377        assert_eq!(rfc3339(1_795_392_000), "2026-11-23T00:00:00Z");
378        assert_eq!(rfc3339(951_782_400), "2000-02-29T00:00:00Z");
379    }
380
381    #[test]
382    fn requirement_is_major_minor() {
383        assert!(runtime_requirement().starts_with(">="));
384    }
385}