use std::collections::VecDeque;
use std::marker::PhantomData;
use super::{
select_rows_keyed_on, select_rows_pool_with_related, ExecError, LoadRelated, MaybeMyFromRow,
MaybeMyLoadRelated, MaybePgFromRow, MaybeSqliteFromRow, MaybeSqliteLoadRelated,
};
use crate::core::{Filter, Model, NullsOrder, Op, OrderItem, SelectQuery, SqlValue};
use crate::sql::Pool;
pub struct ChunkedIter<T> {
pub(super) query: crate::core::SelectQuery,
pub(super) chunk_size: i64,
pub(super) offset: i64,
pub(super) end: Option<i64>,
pub(super) exhausted: bool,
pub(super) buffer: VecDeque<T>,
pub(super) seen: i64,
pub(super) last_key: Option<SqlValue>,
pub(super) _model: PhantomData<fn() -> T>,
}
impl<T> ChunkedIter<T>
where
T: Model
+ MaybePgFromRow
+ MaybeMyFromRow
+ MaybeSqliteFromRow
+ LoadRelated
+ MaybeMyLoadRelated
+ MaybeSqliteLoadRelated
+ Send
+ Unpin,
{
pub async fn next_chunk(&mut self, pool: &Pool) -> Result<Option<Vec<T>>, ExecError> {
if !self.buffer.is_empty() {
let buffered: Vec<T> = self.buffer.drain(..).collect();
self.seen += buffered.len() as i64;
return Ok(Some(buffered));
}
match self.fetch_next_chunk(pool).await? {
Some(rows) => {
self.seen += rows.len() as i64;
Ok(Some(rows))
}
None => Ok(None),
}
}
pub async fn next_row(&mut self, pool: &Pool) -> Result<Option<T>, ExecError> {
if let Some(row) = self.buffer.pop_front() {
self.seen += 1;
return Ok(Some(row));
}
match self.fetch_next_chunk(pool).await? {
Some(rows) => {
self.buffer.extend(rows);
let row = self.buffer.pop_front();
if row.is_some() {
self.seen += 1;
}
Ok(row)
}
None => Ok(None),
}
}
async fn fetch_next_chunk(&mut self, pool: &Pool) -> Result<Option<Vec<T>>, ExecError> {
if self.exhausted {
return Ok(None);
}
let size = self.end.map_or(self.chunk_size, |end| {
self.chunk_size.min(end - self.offset)
});
if size <= 0 {
self.exhausted = true;
return Ok(None);
}
let rows = if super::in_block(pool) {
self.keyset_chunk(pool, size).await?
} else {
let mut q = self.query.clone();
let (limit, offset) = if q.compound.is_empty() {
(&mut q.limit, &mut q.offset)
} else {
(&mut q.compound_limit, &mut q.compound_offset)
};
*limit = Some(size);
*offset = Some(self.offset);
select_rows_pool_with_related::<T>(pool, &q).await?
};
let n = rows.len() as i64;
if rows.is_empty() {
self.exhausted = true;
return Ok(None);
}
if n < size {
self.exhausted = true;
}
self.offset += n;
Ok(Some(rows))
}
async fn keyset_chunk(&mut self, pool: &Pool, size: i64) -> Result<Vec<T>, ExecError> {
let refused = ExecError::InAtomic {
op: "`iterator()` not ordered by the primary key",
reason: "rows the loop changes could be skipped; order by the primary key only \
and keep it in the projection",
};
let mut q: SelectQuery = self.query.clone();
let Some(pk) = q.model.primary_key().filter(|pk| {
q.compound.is_empty() && q.projection.as_ref().is_none_or(|p| p.contains(&pk.column))
}) else {
return Err(refused);
};
let desc = match q.order_by.as_slice() {
[] => false,
[OrderItem::Column {
column,
desc,
nulls: NullsOrder::Default,
}] if *column == pk.column => *desc,
_ => return Err(refused),
};
q.order_by = vec![OrderItem::Column {
column: pk.column,
desc,
nulls: NullsOrder::Default,
}];
q.limit = Some(size);
match self.last_key.clone() {
None => q.offset = Some(self.offset),
Some(key) => {
q.offset = None;
let op = if desc { Op::Lt } else { Op::Gt };
q.where_clause
.push_and(Filter::new(pk.column, op, key).into());
}
}
let rows = select_rows_keyed_on::<T, _>(pool, &q, (pk.column, Some(pk.ty))).await?;
if let Some((key, _)) = rows.last() {
self.last_key = Some(key.clone());
}
Ok(rows.into_iter().map(|(_, row)| row).collect())
}
#[must_use]
pub fn rows_seen(&self) -> i64 {
self.seen
}
#[must_use]
pub fn is_exhausted(&self) -> bool {
self.exhausted && self.buffer.is_empty()
}
}