dspk_pipeline/lib.rs
1//! dspk pipelines as data: a versioned node-graph spec ([`PipelineSpec`]) and an executor
2//! ([`run`]) that streams a recording through dspk-ops stages on the device.
3//!
4//! ```json
5//! {"dspk_pipeline": 1,
6//! "nodes": {"raw": {"op": "read", "path": "recording.bin"},
7//! "hp": {"op": "iir", "band": "highpass", "cutoff": [300.0], "mode": "forward"},
8//! "out": {"op": "write", "path": "filtered.zarr", "format": "zarr"}},
9//! "edges": [["raw", "hp"], ["hp", "out"]]}
10//! ```
11//!
12//! - JSON is canonical; TOML is accepted for hand-written files ([`PipelineSpec::load`]); the JSON
13//! Schema of the spec is [`PipelineSpec::json_schema`].
14//! - The executor reads windows with read-ahead, uploads each window once (integer recordings as
15//! stored integers, scaled on the device), runs every stage on the device, downloads once and
16//! hands the result to a writer thread (raw, or Zarr chunks directly); the canonical spec is
17//! written next to the output as provenance (`<output>.pipeline.json`). The report counts the
18//! bytes moved at each step and the time spent in each ([`StageTimings`]).
19//! - One spec runs over many inputs (globs, lists) and may read several recordings as one (merge).
20//! - [`plan`] says what a run would do (mode, halos, windows, bytes) without touching the device;
21//! [`run_with`] reports progress after each window and can cancel.
22
23mod fit;
24mod prepare;
25mod run;
26mod spec;
27
28pub use fit::{fit_whitening, unfitted_whitening, WhitenFit};
29pub use prepare::{default_open, plan, plan_with, ExecutedMode, Plan};
30pub use run::{events_csv, events_frame, read_events_zarr, run, run_with, write_events, write_events_zarr, EVENTS_VERSION, DataMovement, Progress, RunReport, StageTimings, PROVENANCE_SUFFIX, WRITE_QUEUE_WINDOWS};
31pub use spec::{
32 Band, Chain, Compression, Design, Direction, Edge, ExecMode, Execution, Input, MergePart, Mode, Node, OutputFormat, PipelineSpec, Start, Upload,
33 DEFAULT_BLOSC_LEVEL, DEFAULT_CHUNK_SAMPLES, DEFAULT_ZSTD_LEVEL, SPEC_VERSION, WHITEN_DEFAULT_CHUNKS, WHITEN_DEFAULT_FIT_S,
34};
35
36/// Why a pipeline could not be read, checked or run.
37#[derive(Debug, thiserror::Error)]
38pub enum PipelineError {
39 #[error("pipeline spec: {0}")]
40 Parse(String),
41 #[error("invalid pipeline: {0}")]
42 Invalid(String),
43 #[error("{0}")]
44 Io(String),
45 #[error(transparent)]
46 Dsp(#[from] dspk_core::DspError),
47 #[error("compute runtime: {0}")]
48 Compute(String),
49 #[error("cancelled")]
50 Cancelled,
51}