mariadb-cli 0.1.1

One-shot agent-native command line client for MariaDB servers
Documentation
//! The single output pipeline, shared by every command.
//!
//! Projection, filtering, ordering, deduplication, ceilings and counting are
//! implemented here once and applied to any payload. No command reimplements
//! them, and no command may skip them.
//!
//! The order is fixed and never varies:
//! filter, sort, dedupe, limit, select, count-only, truncate cell, volume
//! ceiling. Cutting happens before serialisation, so an oversized answer never
//! exists in memory in the first place.
//!
//! # The same operations, one row at a time
//!
//! [`Pipeline::apply`] works on a payload that is already gathered, so it can
//! only serve an answer that fits in memory. [`Pipeline::flow`] applies the
//! same operations to one arriving row at a time, which is what lets a read
//! larger than memory still be filtered, projected, deduplicated and cut. The
//! two share their helpers on purpose: a row must not mean one thing gathered
//! and another in flight.
//!
//! Two operations cannot be served that way, and [`Pipeline::accumulates`]
//! names them. Ordering the whole result needs every row before the first one
//! can be placed, and counting the whole result is a single number over a set
//! that is not a set until it is complete. Both are answered by gathering, and
//! the caller is told so rather than left to guess.

mod apply;
mod filter;
mod flow;
mod summary;
mod value;

pub use filter::Filter;
pub use flow::{RowFlow, Step};

/// Everything the caller asked the pipeline to do.
#[derive(Debug, Clone, Default)]
pub struct Pipeline {
    /// Keep only these dotted paths in each element.
    pub select: Vec<String>,
    /// Keep only elements passing every test.
    pub filters: Vec<Filter>,
    /// Order ascending by this dotted path.
    pub sort: Option<String>,
    /// Drop later elements repeating this path's value.
    pub dedupe_by: Option<String>,
    /// How many elements to pass over before the ceiling applies.
    ///
    /// The resume mark of FR-033, and a plain offset rather than an encoded
    /// blob: the design dropped the opaque cursor precisely because a caller
    /// could neither predict nor construct one. It is applied after filtering,
    /// ordering and deduplication, so two pages of the same request describe
    /// the same set and no row is seen twice or missed.
    pub cursor: Option<usize>,
    /// Ceiling on emitted elements.
    pub limit: Option<usize>,
    /// Replace the payload with a count.
    pub count_only: bool,
    /// Replace the payload with a profile of its columns.
    ///
    /// FR-040: a caller who wants to understand a set without reading it gets,
    /// per column, how many rows carried a value, how many were absent, how
    /// many distinct values appeared and a small sample. The profile is always
    /// smaller than the set it describes, which is the whole point.
    pub summary: bool,
    /// Replace credential-shaped text inside the answer with a fixed mark.
    ///
    /// Off unless asked for, and deliberately so. Returning the rows a caller
    /// selected is what this product is for, and a client that quietly altered
    /// them would be answering a different question from the one asked. What
    /// was missing was the option, not the default: the redaction this product
    /// already performs covers the diagnostic channel and the statement text of
    /// `server processes`, and never once touched the contents of a cell, so a
    /// `SELECT` over a table of credentials put every one of them on a channel
    /// documented as carrying data and nothing else.
    pub redact: bool,
    /// Shorten any string above this many characters.
    pub truncate_cell: Option<usize>,
    /// Ceiling on total volume, enforced by dropping whole elements.
    pub max_output_bytes: Option<usize>,
    /// How many of the most frequent values a profile shows per column.
    ///
    /// Carried here rather than written into the profiling code, which is where
    /// it was. The argument beside it there was that a caller who raised the row
    /// ceiling to pull a large table did not thereby ask for a thousand-entry
    /// frequency table, and that argument is right: this is not derived from the
    /// row ceiling. It is a number the reader is entitled to choose, and
    /// `output.most_frequent_shown` is where they choose it.
    ///
    /// `None` means the built-in, which is what a pipeline built by [`Default`]
    /// carries. It deliberately does NOT mean "every distinct value": a profile
    /// must not grow with the set it describes, and a proof beside the
    /// profiling code measures that. Trying `None` as unbounded failed it at
    /// 299130 bytes for ten thousand rows against 495 for ten.
    pub most_frequent_shown: Option<usize>,
}

impl Pipeline {
    /// Whether the pipeline would leave the payload untouched.
    #[must_use]
    pub fn is_identity(&self) -> bool {
        self.select.is_empty()
            && self.filters.is_empty()
            && self.sort.is_none()
            && self.dedupe_by.is_none()
            && self.cursor.is_none()
            && self.limit.is_none()
            && !self.count_only
            && !self.summary
            && !self.redact
            && self.truncate_cell.is_none()
            && self.max_output_bytes.is_none()
    }

    /// Whether answering this pipeline needs the whole result gathered first.
    ///
    /// Ordering the whole result cannot begin until the last row has arrived,
    /// because the first row of the answer may be the last one the server
    /// sends. Counting the whole result is a single number over a set that is
    /// not a set until it is complete. Every other operation decides one row
    /// on its own and therefore streams.
    #[must_use]
    pub const fn accumulates(&self) -> bool {
        !self.accumulated_for().is_empty()
    }

    /// The operations of this pipeline that need the whole set gathered.
    ///
    /// Named rather than counted, because a caller who asked for a flow and
    /// received a gathered answer has to know which operation to drop to get
    /// the flow back.
    #[must_use]
    pub const fn accumulated_for(&self) -> &'static [&'static str] {
        match (self.sort.is_some(), self.count_only, self.summary) {
            (true, true, _) => &["sort", "count-only"],
            (true, false, true) => &["sort", "summary"],
            (true, false, false) => &["sort"],
            (false, true, _) => &["count-only"],
            (false, false, true) => &["summary"],
            (false, false, false) => &[],
        }
    }

    /// The columns a projection leaves, in the order the projection asked for.
    ///
    /// A path survives only when its first segment names a declared column,
    /// because a list that announced a path no row carries would describe a
    /// column the answer never had. A caller building a table from that list
    /// would render an empty column the data never held.
    ///
    /// The gathered answer and the flow both name their columns through this,
    /// so one request cannot describe two different shapes depending on which
    /// path served it.
    #[must_use]
    pub fn columns_after_projection(&self, declared: &[String]) -> Vec<String> {
        if self.select.is_empty() {
            return declared.to_vec();
        }
        self.select
            .iter()
            .filter(|path| {
                let path = path.as_str();
                let head = path.split('.').next().unwrap_or(path);
                declared.iter().any(|name| name == head)
            })
            .cloned()
            .collect()
    }

    /// The same operations, ready to be applied one arriving row at a time.
    ///
    /// The returned value carries no rows. What it keeps is bounded by the
    /// operations asked for and never by how many rows arrive, except for the
    /// keys deduplication has to remember, which is what deduplication means.
    #[must_use]
    pub fn flow(&self) -> RowFlow<'_> {
        self.flow_charging(0)
    }

    /// The same flow, with the bytes of the answer's own frame already charged.
    ///
    /// FR-039 promises a reply inside the volume ceiling, and a flow spends
    /// that ceiling on rows alone unless somebody tells it what the rows are
    /// wrapped in. The wrapping is written by the shape, not by the pipeline —
    /// the opening of the document, the declared columns, the closing count and
    /// the report of what was done — so its size is measured where it is
    /// written and handed in here. A flow opened without it overshoots by
    /// exactly the size of its own frame, on every answer.
    ///
    /// # Errors
    ///
    /// Returns a limit failure when the frame alone is already over the
    /// ceiling: there is no first row that could be emitted inside it, and a
    /// document cut before it opens parses as nothing.
    pub fn flow_within(&self, frame: usize) -> crate::error::Result<RowFlow<'_>> {
        if self.max_output_bytes.is_some_and(|ceiling| frame > ceiling) {
            let ceiling = self.max_output_bytes.unwrap_or_default();
            return Err(crate::error::Failure::limit(
                "limit.output_bytes",
                format!("the answer needs {frame} bytes and the volume ceiling is {ceiling}"),
            )
            .with_detail("detail", format!("{frame} > {ceiling}"))
            .with_hint("raise --max-output-bytes, or ask for fewer columns"));
        }
        Ok(self.flow_charging(frame))
    }

    /// The same flow, with the frame the SHAPE will write around it charged.
    ///
    /// The frame is written by the formatter and by nobody else, so it is
    /// measured by writing it: the opening and the closing are produced once
    /// into a buffer that is thrown away, and what they cost is what is
    /// charged. Measuring beats estimating here because a constant would
    /// silently stop being true the day a shape changes what it writes, and the
    /// failure would look like a ceiling that is merely a little generous.
    ///
    /// Both ends are written, never just the opening: an opening charged alone
    /// reproduces the very defect this repairs, with a smaller number and the
    /// appearance of being fixed.
    ///
    /// The closing is measured with the gaps the caller could end up declaring
    /// and BOTH ways the flow can end, because a shape may write more when the
    /// set was cut and more when it was not. The line-oriented shapes write no
    /// closing record at all for a cut set, since there the record IS the claim
    /// that the set is whole; measuring only that case charged them nothing and
    /// left the closing record of a whole answer over the ceiling. The document
    /// shape is the other way around, carrying one extra field when cut. So the
    /// larger of the two is charged, which is the only figure that holds
    /// whichever way the flow turns out to end.
    ///
    /// # Errors
    ///
    /// Returns a limit failure when the frame alone is over the ceiling, and
    /// whatever failure the shape produced while being measured.
    pub fn flow_framed(
        &self,
        formatter: &mut dyn crate::output::format::Formatter,
        tail: &crate::output::Envelope,
        columns: &[String],
        rows: usize,
        gaps: &[crate::output::envelope::Gap],
    ) -> crate::error::Result<RowFlow<'_>> {
        if self.max_output_bytes.is_none() {
            return Ok(self.flow());
        }
        let mut measured = Vec::new();
        formatter.begin_stream(&mut measured, tail, columns)?;
        let opening = measured.len();

        let cut = self.widest_report(rows);
        let mut whole = cut.clone();
        whole.output_truncated = false;

        let emitted = rows as u64;
        formatter.end_stream(&mut measured, tail, emitted, gaps, Some(&cut))?;
        let after_cut = measured.len();
        formatter.end_stream(&mut measured, tail, emitted, gaps, Some(&whole))?;

        let closing = (after_cut - opening).max(measured.len() - after_cut);
        self.flow_within(opening + closing)
    }

    /// A flow that has already spent `charged` bytes of the volume ceiling.
    fn flow_charging(&self, charged: usize) -> RowFlow<'_> {
        RowFlow {
            pipeline: self,
            seen: std::collections::BTreeSet::new(),
            passed_over: 0,
            offered: 0,
            emitted: 0,
            bytes: charged,
            whole: true,
            shortened: false,
            redacted: false,
            over_the_ceiling: false,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A projection names the columns it kept, in the order it asked for them,
    /// and never a path the answer did not declare.
    #[test]
    fn a_projection_names_what_it_kept_and_nothing_it_invented() {
        let declared = vec!["a".to_owned(), "b".to_owned(), "c".to_owned()];

        let idle = Pipeline::default();
        assert_eq!(idle.columns_after_projection(&declared), declared);

        let projecting = Pipeline {
            select: vec!["c".to_owned(), "a".to_owned(), "nowhere".to_owned()],
            ..Pipeline::default()
        };
        assert_eq!(
            projecting.columns_after_projection(&declared),
            ["c", "a"],
            "the order asked for is kept, and the path nothing declared is dropped"
        );
    }

    /// A path reaching inside a column is carried by the column it reaches
    /// into, so dropping it would hide a value the rows really do carry.
    #[test]
    fn a_path_reaching_into_a_declared_column_survives() {
        let declared = vec!["document".to_owned()];
        let pipeline = Pipeline {
            select: vec!["document.total".to_owned()],
            ..Pipeline::default()
        };
        assert_eq!(
            pipeline.columns_after_projection(&declared),
            ["document.total"]
        );
    }

    /// Only an answer that is a property of the whole set may gather.
    #[test]
    fn only_an_order_or_a_count_needs_the_whole_set() {
        assert!(Pipeline {
            sort: Some("id".to_owned()),
            ..Pipeline::default()
        }
        .accumulates());
        assert!(Pipeline {
            count_only: true,
            ..Pipeline::default()
        }
        .accumulates());

        for streaming in [
            Pipeline {
                limit: Some(10),
                ..Pipeline::default()
            },
            Pipeline {
                filters: vec![Filter::parse("a=b").expect("parses")],
                ..Pipeline::default()
            },
            Pipeline {
                select: vec!["a".to_owned()],
                ..Pipeline::default()
            },
            Pipeline {
                dedupe_by: Some("a".to_owned()),
                ..Pipeline::default()
            },
            Pipeline {
                truncate_cell: Some(10),
                ..Pipeline::default()
            },
            Pipeline {
                max_output_bytes: Some(10),
                ..Pipeline::default()
            },
        ] {
            assert!(
                !streaming.accumulates(),
                "{streaming:?} decides one row at a time"
            );
        }
    }
}