use super::value::{lookup_row, project_row, redact_in_place, scalar_text, truncate_in_place};
use super::Pipeline;
use crate::output::envelope::Row;
#[derive(Debug)]
pub enum Step {
Emit(Row),
Skip,
Stop,
}
#[allow(clippy::struct_excessive_bools)]
pub struct RowFlow<'a> {
pub(super) pipeline: &'a Pipeline,
pub(super) seen: std::collections::BTreeSet<String>,
pub(super) passed_over: usize,
pub(super) offered: usize,
pub(super) emitted: usize,
pub(super) bytes: usize,
pub(super) whole: bool,
pub(super) shortened: bool,
pub(super) redacted: bool,
pub(super) over_the_ceiling: bool,
}
impl RowFlow<'_> {
pub fn offer(&mut self, row: Row) -> Step {
self.offered += 1;
if !self
.pipeline
.filters
.iter()
.all(|filter| filter.accepts_row(&row))
{
return Step::Skip;
}
if let Some(path) = &self.pipeline.dedupe_by {
if let Some(found) = lookup_row(&row, path) {
if !self.seen.insert(scalar_text(found).into_owned()) {
return Step::Skip;
}
}
}
if self
.pipeline
.cursor
.is_some_and(|mark| self.passed_over < mark)
{
self.passed_over += 1;
return Step::Skip;
}
if self
.pipeline
.limit
.is_some_and(|ceiling| self.emitted >= ceiling)
{
self.over_the_ceiling = true;
return Step::Stop;
}
let mut row = if self.pipeline.select.is_empty() {
row
} else {
project_row(&row, &self.pipeline.select)
};
if self.pipeline.redact {
for value in row.values_mut() {
self.redacted |= redact_in_place(value);
}
}
if let Some(ceiling) = self.pipeline.truncate_cell {
for value in row.values_mut() {
self.shortened |= truncate_in_place(value, ceiling);
}
}
if let Some(ceiling) = self.pipeline.max_output_bytes {
let size = serde_json::to_string(&row).map_or(0, |text| text.len() + 1);
if self.bytes + size > ceiling {
self.whole = false;
return Step::Stop;
}
self.bytes += size;
}
self.emitted += 1;
Step::Emit(row)
}
#[must_use]
pub const fn is_whole(&self) -> bool {
self.whole
}
#[must_use]
pub const fn emitted(&self) -> usize {
self.emitted
}
#[must_use]
pub const fn held_rows(&self) -> usize {
0
}
#[must_use]
pub fn tracked_keys(&self) -> usize {
self.seen.len()
}
#[must_use]
pub fn applied(&self) -> Option<crate::output::envelope::Applied> {
if self.pipeline.is_identity() {
return None;
}
let mut applied = crate::output::envelope::Applied {
input_count: Some(self.offered),
output_count: Some(self.emitted),
content_truncated: self.shortened,
content_redacted: self.redacted,
accumulated_for: self.pipeline.accumulated_for(),
..crate::output::envelope::Applied::default()
};
let dropped = self.offered.saturating_sub(self.emitted);
if dropped > 0 {
applied.dropped = Some(dropped);
}
if !self.whole {
applied.output_truncated = true;
}
if self.pipeline.limit.is_some() {
applied.has_more = Some(self.over_the_ceiling);
} else if self.pipeline.cursor.is_some() {
applied.has_more = Some(false);
}
if self.pipeline.cursor.is_some() || self.pipeline.limit.is_some() {
applied.next_cursor = Some(self.pipeline.cursor.unwrap_or(0) + self.emitted);
}
Some(applied)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::output::ops::Filter;
use serde_json::json;
fn wire_row(index: i64, engine: &str) -> Row {
let mut row = Row::new();
row.insert("id".to_owned(), json!(index));
row.insert("engine".to_owned(), json!(engine));
row
}
fn flowed(pipeline: &Pipeline, rows: impl Iterator<Item = Row>) -> (Vec<Row>, bool) {
let mut stage = pipeline.flow();
let mut kept = Vec::new();
let mut whole = true;
for row in rows {
match stage.offer(row) {
Step::Emit(shaped) => kept.push(shaped),
Step::Skip => {}
Step::Stop => {
whole = stage.is_whole();
break;
}
}
}
(kept, whole)
}
#[test]
fn a_filter_selects_the_same_rows_gathered_or_in_flight() {
let pipeline = Pipeline {
filters: vec![Filter::parse("engine=InnoDB").expect("parses")],
..Pipeline::default()
};
let arriving = vec![
wire_row(1, "InnoDB"),
wire_row(2, "Aria"),
wire_row(3, "InnoDB"),
];
let (kept, _) = flowed(&pipeline, arriving.into_iter());
assert_eq!(kept.len(), 2);
assert_eq!(kept[0]["id"], json!(1));
assert_eq!(kept[1]["id"], json!(3));
}
#[test]
fn the_row_ceiling_stops_the_pull_and_leaves_the_flow_whole() {
let pipeline = Pipeline {
limit: Some(3),
..Pipeline::default()
};
let mut stage = pipeline.flow();
let mut kept = 0;
let mut offered = 0;
for index in 0..1_000_000 {
offered += 1;
match stage.offer(wire_row(index, "InnoDB")) {
Step::Emit(_) => kept += 1,
Step::Skip => {}
Step::Stop => break,
}
}
assert_eq!(kept, 3);
assert_eq!(
offered, 4,
"one row past the ceiling ends the flow, no more"
);
assert!(stage.is_whole(), "the ceiling is what the caller asked for");
}
#[test]
fn a_projection_keeps_only_the_asked_paths() {
let pipeline = Pipeline {
select: vec!["id".to_owned(), "absent".to_owned()],
..Pipeline::default()
};
let (kept, _) = flowed(&pipeline, std::iter::once(wire_row(1, "InnoDB")));
assert_eq!(kept[0].len(), 1);
assert!(kept[0].contains_key("id"));
assert!(!kept[0].contains_key("absent"));
}
#[test]
fn deduplication_carries_keys_and_never_rows() {
let pipeline = Pipeline {
dedupe_by: Some("engine".to_owned()),
..Pipeline::default()
};
let mut stage = pipeline.flow();
let mut kept = 0;
for index in 0..10_000 {
if matches!(stage.offer(wire_row(index, "InnoDB")), Step::Emit(_)) {
kept += 1;
}
}
assert_eq!(kept, 1, "one value, one row");
assert_eq!(
stage.tracked_keys(),
1,
"one key remembered, not ten thousand"
);
assert_eq!(stage.held_rows(), 0);
}
#[test]
fn deduplication_keeps_the_keyless_without_remembering_it() {
let pipeline = Pipeline {
dedupe_by: Some("absent".to_owned()),
..Pipeline::default()
};
let mut stage = pipeline.flow();
for index in 0..100 {
assert!(matches!(
stage.offer(wire_row(index, "InnoDB")),
Step::Emit(_)
));
}
assert_eq!(stage.tracked_keys(), 0);
}
#[test]
fn the_volume_ceiling_stops_the_flow_and_declares_it_cut() {
let pipeline = Pipeline {
max_output_bytes: Some(60),
..Pipeline::default()
};
let arriving = (0..100).map(|index| wire_row(index, "InnoDB"));
let (kept, whole) = flowed(&pipeline, arriving);
assert!(!kept.is_empty(), "what fits is still delivered");
assert!(kept.len() < 100);
assert!(!whole, "rows the caller asked to read were left behind");
}
#[test]
fn the_mark_passes_over_the_same_rows_gathered_or_in_flight() {
let pipeline = Pipeline {
cursor: Some(2),
limit: Some(2),
..Pipeline::default()
};
let arriving = (0..6).map(|index| wire_row(index, "InnoDB"));
let (kept, _) = flowed(&pipeline, arriving);
let seen: Vec<i64> = kept
.iter()
.map(|row| row["id"].as_i64().expect("number"))
.collect();
assert_eq!(seen, [2, 3], "the mark passes over the first two rows");
}
#[test]
fn a_long_flow_holds_no_rows_from_the_first_to_the_last() {
let pipeline = Pipeline {
filters: vec![Filter::parse("engine=InnoDB").expect("parses")],
limit: Some(1_000_000),
select: vec!["id".to_owned()],
truncate_cell: Some(8),
..Pipeline::default()
};
let mut stage = pipeline.flow();
for index in 0..200_000 {
let _ = stage.offer(wire_row(index, "InnoDB"));
assert_eq!(stage.held_rows(), 0);
assert_eq!(
stage.tracked_keys(),
0,
"no deduplication, nothing remembered"
);
}
assert_eq!(stage.emitted(), 200_000);
}
}