dspk-pipeline 0.0.10

dspk pipelines as data: a versioned node-graph spec (JSON, or TOML to write by hand) and an executor that streams a recording through dspk-ops stages on the device
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

//! dspk pipelines as data: a versioned node-graph spec ([`PipelineSpec`]) and an executor
//! ([`run`]) that streams a recording through dspk-ops stages on the device.
//!
//! ```json
//! {"dspk_pipeline": 1,
//!  "nodes": {"raw": {"op": "read", "path": "recording.bin"},
//!            "hp":  {"op": "iir", "band": "highpass", "cutoff": [300.0], "mode": "forward"},
//!            "out": {"op": "write", "path": "filtered.zarr", "format": "zarr"}},
//!  "edges": [["raw", "hp"], ["hp", "out"]]}
//! ```
//!
//! - JSON is canonical; TOML is accepted for hand-written files ([`PipelineSpec::load`]); the JSON
//!   Schema of the spec is [`PipelineSpec::json_schema`].
//! - The executor reads windows with read-ahead, uploads each window once (integer recordings as
//!   stored integers, scaled on the device), runs every stage on the device, downloads once and
//!   hands the result to a writer thread (raw, or Zarr chunks directly); the canonical spec is
//!   written next to the output as provenance (`<output>.pipeline.json`). The report counts the
//!   bytes moved at each step and the time spent in each ([`StageTimings`]).
//! - One spec runs over many inputs (globs, lists) and may read several recordings as one (merge).
//! - [`plan`] says what a run would do (mode, halos, windows, bytes) without touching the device;
//!   [`run_with`] reports progress after each window and can cancel.

mod fit;
mod prepare;
mod run;
mod spec;

pub use fit::{fit_whitening, unfitted_whitening, WhitenFit};
pub use prepare::{default_open, plan, plan_with, ExecutedMode, Plan};
pub use run::{events_csv, events_frame, read_events_zarr, run, run_with, write_events, write_events_zarr, EVENTS_VERSION, DataMovement, Progress, RunReport, StageTimings, PROVENANCE_SUFFIX, WRITE_QUEUE_WINDOWS};
pub use spec::{
    Band, Chain, Compression, Design, Direction, Edge, ExecMode, Execution, Input, MergePart, Mode, Node, OutputFormat, PipelineSpec, Start, Upload,
    DEFAULT_BLOSC_LEVEL, DEFAULT_CHUNK_SAMPLES, DEFAULT_ZSTD_LEVEL, SPEC_VERSION, WHITEN_DEFAULT_CHUNKS, WHITEN_DEFAULT_FIT_S,
};

/// Why a pipeline could not be read, checked or run.
#[derive(Debug, thiserror::Error)]
pub enum PipelineError {
    #[error("pipeline spec: {0}")]
    Parse(String),
    #[error("invalid pipeline: {0}")]
    Invalid(String),
    #[error("{0}")]
    Io(String),
    #[error(transparent)]
    Dsp(#[from] dspk_core::DspError),
    #[error("compute runtime: {0}")]
    Compute(String),
    #[error("cancelled")]
    Cancelled,
}