use crate::error::Error;
use crate::jsonl::json_lines;
use crate::typed_doc::{JsonDoc, TypedDocRef};
use crate::view::JsonView;
use crate::view_list::ViewList;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub struct StreamPos {
pub source_index: usize,
pub record_index: usize,
}
#[derive(Debug, Clone, Default)]
pub struct JsonlDecoder {
carry: Vec<u8>,
}
impl JsonlDecoder {
#[inline]
pub fn new() -> Self {
Self { carry: Vec::new() }
}
#[inline]
pub fn with_capacity(cap: usize) -> Self {
Self {
carry: Vec::with_capacity(cap),
}
}
#[inline]
pub fn pending(&self) -> &[u8] {
&self.carry
}
#[inline]
pub fn clear(&mut self) {
self.carry.clear();
}
#[inline]
pub fn push(&mut self, chunk: &[u8]) {
if chunk.is_empty() {
return;
}
self.carry.extend_from_slice(chunk);
}
pub fn for_each_ready<F>(&mut self, mut f: F) -> Result<(), Error>
where
F: FnMut(&[u8]) -> Result<(), Error>,
{
let mut consumed = 0usize;
let mut start = 0usize;
let buf = self.carry.as_slice();
while start < buf.len() {
let rel = match buf[start..].iter().position(|&b| b == b'\n') {
Some(i) => i,
None => break,
};
let end = start + rel;
let mut line = &buf[start..end];
if line.last() == Some(&b'\r') {
line = &line[..line.len() - 1];
}
if !line.is_empty() {
f(line)?;
}
start = end + 1;
consumed = start;
}
if consumed > 0 {
self.carry.drain(..consumed);
}
Ok(())
}
#[inline]
pub fn push_and_for_each<F>(&mut self, chunk: &[u8], f: F) -> Result<(), Error>
where
F: FnMut(&[u8]) -> Result<(), Error>,
{
self.push(chunk);
self.for_each_ready(f)
}
pub fn finish<F>(&mut self, mut f: F) -> Result<bool, Error>
where
F: FnMut(&[u8]) -> Result<(), Error>,
{
self.for_each_ready(&mut f)?;
if self.carry.is_empty() {
return Ok(false);
}
let mut line = self.carry.as_slice();
if line.last() == Some(&b'\r') {
line = &line[..line.len() - 1];
}
if line.is_empty() {
self.carry.clear();
return Ok(false);
}
let owned = line.to_vec();
self.carry.clear();
f(&owned)?;
Ok(true)
}
pub fn for_each_view_ready<T, F>(&mut self, mut f: F) -> Result<(), Error>
where
T: JsonView,
F: FnMut(T) -> Result<(), Error>,
{
self.for_each_ready(|line| f(T::read_from(line)?))
}
}
pub fn for_each_jsonl_multi<I, B, F>(sources: I, mut f: F) -> Result<(), Error>
where
I: IntoIterator<Item = B>,
B: AsRef<[u8]>,
F: FnMut(StreamPos, &[u8]) -> Result<(), Error>,
{
for (source_index, src) in sources.into_iter().enumerate() {
let buf = src.as_ref();
let mut record_index = 0usize;
for line in json_lines(buf) {
f(
StreamPos {
source_index,
record_index,
},
line,
)?;
record_index += 1;
}
}
Ok(())
}
pub fn for_each_jsonl_view_multi<T, I, B, F>(sources: I, mut f: F) -> Result<(), Error>
where
T: JsonView,
I: IntoIterator<Item = B>,
B: AsRef<[u8]>,
F: FnMut(StreamPos, T) -> Result<(), Error>,
{
for_each_jsonl_multi(sources, |pos, line| f(pos, T::read_from(line)?))
}
pub fn read_jsonl_multi<'a, T: JsonView + 'a>(
sources: impl IntoIterator<Item = &'a [u8]>,
) -> impl Iterator<Item = Result<(StreamPos, T), Error>> + 'a {
let sources: Vec<&'a [u8]> = sources.into_iter().collect();
sources
.into_iter()
.enumerate()
.flat_map(|(source_index, buf)| {
json_lines(buf)
.enumerate()
.map(move |(record_index, line)| {
let pos = StreamPos {
source_index,
record_index,
};
T::read_from(line).map(|t| (pos, t))
})
})
}
pub fn for_each_page_items<T, I, B, F>(
pages: I,
items_path: &str,
mut f: F,
) -> Result<(), Error>
where
T: JsonView,
I: IntoIterator<Item = B>,
B: AsRef<[u8]>,
F: FnMut(StreamPos, T) -> Result<(), Error>,
{
for (source_index, page) in pages.into_iter().enumerate() {
let bytes = page.as_ref();
let doc = TypedDocRef::from_slice(bytes);
let list = ViewList::<T>::from_doc(&doc, items_path)?;
let mut record_index = 0usize;
list.each(|row| {
let pos = StreamPos {
source_index,
record_index,
};
record_index += 1;
f(pos, row)
})?;
}
Ok(())
}
pub fn for_each_page_array<T, I, B, F>(pages: I, mut f: F) -> Result<(), Error>
where
T: JsonView,
I: IntoIterator<Item = B>,
B: AsRef<[u8]>,
F: FnMut(StreamPos, T) -> Result<(), Error>,
{
for (source_index, page) in pages.into_iter().enumerate() {
let bytes = page.as_ref();
let list = ViewList::<T>::from_array_bytes(bytes)?;
let mut record_index = 0usize;
list.each(|row| {
let pos = StreamPos {
source_index,
record_index,
};
record_index += 1;
f(pos, row)
})?;
}
Ok(())
}
pub fn for_each_page_doc<I, B, F>(pages: I, mut f: F) -> Result<(), Error>
where
I: IntoIterator<Item = B>,
B: AsRef<[u8]>,
F: FnMut(StreamPos, TypedDocRef<'_>) -> Result<(), Error>,
{
for (source_index, page) in pages.into_iter().enumerate() {
let bytes = page.as_ref();
let doc = TypedDocRef::from_slice(bytes);
f(
StreamPos {
source_index,
record_index: 0,
},
doc,
)?;
}
Ok(())
}
pub fn for_each_page_item_raw<I, B, F>(
pages: I,
items_path: &str,
mut f: F,
) -> Result<(), Error>
where
I: IntoIterator<Item = B>,
B: AsRef<[u8]>,
F: FnMut(StreamPos, &[u8]) -> Result<(), Error>,
{
for (source_index, page) in pages.into_iter().enumerate() {
let bytes = page.as_ref();
let doc = TypedDocRef::from_slice(bytes);
let mut record_index = 0usize;
for item in doc.elems(items_path)? {
let span = item?;
f(
StreamPos {
source_index,
record_index,
},
span,
)?;
record_index += 1;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::build::ObjectBuilder;
use crate::convert::{FromJsonSlice, ToJsonBytes};
use crate::path::parse_path;
use crate::scan::find_value;
use crate::upsert_at_path;
struct IdOnly {
id: u64,
}
impl JsonView for IdOnly {
fn read_from(json: &[u8]) -> Result<Self, Error> {
let s = find_value(json, &parse_path("id"))?;
Ok(Self {
id: u64::from_json_slice(s).ok_or(Error::TypeMismatch {
expected: "u64",
found: "bad",
})?,
})
}
fn write_into(&self, json: &mut Vec<u8>) -> Result<(), Error> {
upsert_at_path(json, &parse_path("id"), &self.id.to_json_bytes())
}
}
#[test]
fn multi_jsonl_shards() {
let a = br#"{"id":1}
{"id":2}
"#;
let b = br#"{"id":3}
"#;
let mut ids = Vec::new();
for_each_jsonl_view_multi::<IdOnly, _, _, _>([a.as_slice(), b.as_slice()], |pos, row| {
ids.push((pos.source_index, pos.record_index, row.id));
Ok(())
})
.unwrap();
assert_eq!(ids, vec![(0, 0, 1), (0, 1, 2), (1, 0, 3)]);
}
#[test]
fn jsonl_decoder_chunked() {
let mut d = JsonlDecoder::new();
let mut ids = Vec::new();
d.push(br#"{"id":1}"#);
d.push(b"\n{\"i");
d.for_each_ready(|line| {
ids.push(IdOnly::read_from(line).unwrap().id);
Ok(())
})
.unwrap();
assert_eq!(ids, vec![1]);
assert!(!d.pending().is_empty());
d.push(br#"d":2}"#);
d.push(b"\n");
d.for_each_ready(|line| {
ids.push(IdOnly::read_from(line).unwrap().id);
Ok(())
})
.unwrap();
assert_eq!(ids, vec![1, 2]);
assert!(d.pending().is_empty());
}
#[test]
fn page_items_paginated_api() {
let p1 = ObjectBuilder::new()
.raw_field(
"items",
br#"[{"id":1},{"id":2}]"#,
)
.field("next", &2u64)
.finish();
let p2 = ObjectBuilder::new()
.raw_field("items", br#"[{"id":3}]"#)
.null_field("next")
.finish();
let mut sum = 0u64;
let mut n = 0usize;
for_each_page_items::<IdOnly, _, _, _>([p1.as_slice(), p2.as_slice()], "items", |pos, row| {
sum += row.id;
n += 1;
assert!(pos.source_index < 2);
Ok(())
})
.unwrap();
assert_eq!(sum, 6);
assert_eq!(n, 3);
}
#[test]
fn page_root_array() {
let pages = [br#"[{"id":10},{"id":11}]"#.as_slice(), br#"[{"id":12}]"#.as_slice()];
let mut ids = Vec::new();
for_each_page_array::<IdOnly, _, _, _>(pages, |_pos, row| {
ids.push(row.id);
Ok(())
})
.unwrap();
assert_eq!(ids, vec![10, 11, 12]);
}
#[test]
fn page_doc_single_objects() {
let pages = [br#"{"id":1,"page":0}"#.as_slice(), br#"{"id":2,"page":1}"#.as_slice()];
let mut ids = Vec::new();
for_each_page_doc(pages, |_pos, doc| {
ids.push(doc.get::<u64>("id").unwrap());
Ok(())
})
.unwrap();
assert_eq!(ids, vec![1, 2]);
}
#[test]
fn decoder_finish_without_newline() {
let mut d = JsonlDecoder::new();
d.push(br#"{"id":9}"#);
let mut got = 0u64;
assert!(d
.finish(|line| {
got = IdOnly::read_from(line).unwrap().id;
Ok(())
})
.unwrap());
assert_eq!(got, 9);
}
}