Skip to main content

openreadout_index/
lib.rs

1//! A catalog of every instrument data set on a file share, as open files the user owns.
2//!
3//! `openreadout index ROOT… -o INDEX_DIR` walks the roots, reads only headers (never pixels,
4//! samples or spectra), and writes three Parquet tables and a manifest into `INDEX_DIR`:
5//!
6//! - `experiments.parquet`: one row per data set (a file, a multi-file set such as a
7//!   multi-file OME-TIFF or a CZI with its parts, or a directory data set such as a Bruker
8//!   `.d`), with the experiment model flattened into columns (sample, instrument, technique
9//!   term, method, operator, start time), image/table/trace/spectra summaries, integrity status
10//!   and personal-data flags;
11//! - `files.parquet`: one row per file seen, pointing at its data set (or `unknown`);
12//! - `problems.parquet`: integrity findings, unreadable files, walk errors and personal-data
13//!   flags;
14//! - `index.json`: [`IndexManifest`]: roots, settings, crawl counters, totals per format,
15//!   family, year and unknown extension, and the column list of every table.
16//!
17//! The crawl is parallel (the rayon pool), resumable (a journal with checkpoints; a killed run
18//! continues where it stopped), incremental (unchanged files are not reopened; moved files are
19//! recognised by their content fingerprint), and its memory does not grow with the size of the
20//! tree (records stream to disk; the walk holds one directory listing per level). See
21//! [`crawl`] for the state files; multi-file grouping happens when the tables are written.
22//!
23//! On top of an index: [`search()`] (a small query language, [`query`]), [`health()`]
24//! (storage health report) and [`export_dataset`] (an ML-ready slice with a datasheet). [`pii`] finds personal
25//! data in header metadata and redacts it. [`report`] builds the privacy-reviewed diagnostic
26//! bundle of `openreadout report` for a file a reader refused or could not validate.
27//!
28//! # Example
29//!
30//! ```no_run
31//! use openreadout_core::Registry;
32//! use openreadout_index::{IndexOptions, index, search, SearchRequest};
33//!
34//! let registry = Registry::new(); // the CLI registers every format reader here
35//! let mut opts = IndexOptions::default();
36//! opts.roots = vec!["/lab/share".into()];
37//! opts.index_dir = "/lab/index".into();
38//! let manifest = index(&registry, &opts, None)?;
39//! println!("{} data sets", manifest.datasets);
40//! let mut req = SearchRequest::default();
41//! req.query = "format=czi objective=63x acquired<2020".into();
42//! let hits = search("/lab/index".as_ref(), &req)?;
43//! println!("{} matches", hits.total);
44//! # Ok::<(), openreadout_core::Error>(())
45//! ```
46#![forbid(unsafe_code)]
47#![warn(missing_docs)]
48
49pub mod crawl;
50pub mod export;
51pub(crate) mod finalize;
52pub mod fingerprint;
53pub mod formats;
54pub mod group;
55pub mod health;
56pub mod manifest;
57pub mod pii;
58pub mod query;
59pub mod record;
60pub mod report;
61pub mod search;
62pub mod tables;
63pub mod walk;
64
65#[doc(hidden)]
66pub mod synth;
67
68pub use crawl::{CheckMode, CrawlProgress, IndexOptions, index};
69pub use export::{ExportDatasetOptions, ExportDatasetReport, export_dataset};
70pub use health::{HealthOptions, HealthReport, health};
71pub use manifest::{IndexManifest, read_manifest};
72pub use search::{SearchOutput, SearchRequest, search};
73
74/// Columns of the table called `name` (`experiments`, `files`, `problems`).
75pub fn tables_named(name: &str) -> &'static [tables::ColumnDef] {
76    match name {
77        "experiments" => tables::EXPERIMENTS,
78        "files" => tables::FILES,
79        "problems" => tables::PROBLEMS,
80        _ => &[],
81    }
82}