use std::io::Write;
use crate::error::Error;
use crate::index::IndexedDocument;
use crate::typed_doc::{JsonDoc, TypedDoc, TypedDocRef};
use crate::view::JsonView;
#[inline]
pub fn json_lines(buf: &[u8]) -> JsonLines<'_> {
JsonLines { rest: buf }
}
#[derive(Debug, Clone)]
pub struct JsonLines<'a> {
rest: &'a [u8],
}
impl<'a> Iterator for JsonLines<'a> {
type Item = &'a [u8];
fn next(&mut self) -> Option<&'a [u8]> {
while !self.rest.is_empty() {
let nl = self
.rest
.iter()
.position(|&b| b == b'\n')
.unwrap_or(self.rest.len());
let mut line = &self.rest[..nl];
self.rest = if nl < self.rest.len() {
&self.rest[nl + 1..]
} else {
&[]
};
if line.last() == Some(&b'\r') {
line = &line[..line.len() - 1];
}
if !line.is_empty() {
return Some(line);
}
}
None
}
}
impl<'a> JsonLines<'a> {
#[inline]
pub fn rest(&self) -> &'a [u8] {
self.rest
}
}
pub fn read_jsonl<'a, T: JsonView + 'a>(
buf: &'a [u8],
) -> impl Iterator<Item = Result<T, Error>> + 'a {
json_lines(buf).map(T::read_from)
}
pub fn read_jsonl_indexed<'a, T: JsonView + 'a>(
buf: &'a [u8],
) -> impl Iterator<Item = Result<T, Error>> + 'a {
json_lines(buf).map(T::read_from_indexed)
}
pub fn read_line_indexed<T: JsonView>(
line: &[u8],
array_paths: &[&str],
) -> Result<T, Error> {
let doc = IndexedDocument::build(line, array_paths)?;
T::read_from_doc(&doc)
}
pub fn jsonl_docs(buf: &[u8]) -> impl Iterator<Item = TypedDocRef<'_>> + '_ {
json_lines(buf).map(TypedDocRef::from_slice)
}
pub fn jsonl_docs_owned(buf: &[u8]) -> impl Iterator<Item = TypedDoc> + '_ {
json_lines(buf).map(TypedDoc::from_slice)
}
pub fn write_jsonl_line(out: &mut impl Write, raw: &[u8]) -> Result<(), Error> {
out.write_all(raw).map_err(io_err)?;
out.write_all(b"\n").map_err(io_err)?;
Ok(())
}
pub fn write_jsonl_views<'a, T: JsonView + 'a>(
out: &mut impl Write,
views: impl IntoIterator<Item = &'a T>,
) -> Result<(), Error> {
for v in views {
let doc = TypedDoc::from_view(v)?;
write_jsonl_line(out, doc.as_bytes())?;
}
Ok(())
}
pub fn write_jsonl_docs<'a>(
out: &mut impl Write,
docs: impl IntoIterator<Item = &'a (impl AsRef<[u8]> + ?Sized + 'a)>,
) -> Result<(), Error> {
for d in docs {
write_jsonl_line(out, d.as_ref())?;
}
Ok(())
}
pub fn for_each_jsonl_line<F>(buf: &[u8], mut f: F) -> Result<(), Error>
where
F: FnMut(&[u8]) -> Result<(), Error>,
{
for line in json_lines(buf) {
f(line)?;
}
Ok(())
}
pub fn for_each_jsonl_doc<F>(buf: &[u8], mut f: F) -> Result<(), Error>
where
F: FnMut(TypedDocRef<'_>) -> Result<(), Error>,
{
for line in json_lines(buf) {
f(TypedDocRef::from_slice(line))?;
}
Ok(())
}
fn io_err(_e: std::io::Error) -> Error {
Error::InvalidJsonSyntax {
pos: 0,
msg: "I/O error writing JSONL",
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::build::ObjectBuilder;
use crate::convert::{FromJsonSlice, ToJsonBytes};
use crate::path::parse_path;
use crate::scan::find_value;
use crate::upsert_at_path;
struct IdOnly {
id: u64,
}
impl JsonView for IdOnly {
fn read_from(json: &[u8]) -> Result<Self, Error> {
let s = find_value(json, &parse_path("id"))?;
Ok(Self {
id: u64::from_json_slice(s).ok_or(Error::TypeMismatch {
expected: "u64",
found: "bad",
})?,
})
}
fn write_into(&self, json: &mut Vec<u8>) -> Result<(), Error> {
upsert_at_path(json, &parse_path("id"), &self.id.to_json_bytes())
}
}
#[test]
fn jsonl_docs_and_write_views() {
let buf = br#"{"id":1}
{"id":2}
"#;
let ids: Vec<u64> = jsonl_docs(buf)
.map(|d| d.get::<u64>("id").unwrap())
.collect();
assert_eq!(ids, vec![1, 2]);
let views = [IdOnly { id: 3 }, IdOnly { id: 4 }];
let mut out = Vec::new();
write_jsonl_views(&mut out, &views).unwrap();
let back: Vec<u64> = read_jsonl::<IdOnly>(&out)
.map(|r| r.unwrap().id)
.collect();
assert_eq!(back, vec![3, 4]);
}
#[test]
fn write_jsonl_docs_from_builder() {
let d1 = ObjectBuilder::new().field("x", &1u64).into_doc();
let d2 = ObjectBuilder::new().field("x", &2u64).into_doc();
let mut out = Vec::new();
write_jsonl_docs(&mut out, [d1.as_bytes(), d2.as_bytes()]).unwrap();
assert_eq!(out, b"{\"x\":1}\n{\"x\":2}\n");
}
}