mod emit;
mod focus;
mod jmespath;
mod plan;
mod select;
mod sink;
mod transform;
pub use jmespath::{parse_jmespath_expr, parse_project_path, select_from_project_path};
pub use plan::{MissingPolicy, ProjectPlan, ProjectStyle};
pub use select::{
ArraySelect, CmpOp, HashField, ObjectSelect, ProjectPathSegment, SelectExpr,
};
pub use sink::{CountingSink, WriteSink};
pub use transform::{Transform, TransformPipeline};
use std::io::Write;
use crate::error::Error;
use crate::index::IndexedDocument;
use crate::path::{parse_path, PathSegment};
use crate::scan::{find_value, find_value_offsets, skip_value, skip_whitespace};
use sink::{CountingSink as CountSink, EmitOut, WriteSink as WSink};
pub fn project(json: &[u8], plan: &ProjectPlan) -> Result<Vec<u8>, Error> {
let mut out = Vec::new();
project_into(json, plan, &mut out)?;
Ok(out)
}
pub fn project_into(json: &[u8], plan: &ProjectPlan, out: &mut Vec<u8>) -> Result<(), Error> {
project_to_sink(json, plan, out)
}
pub fn project_indexed(doc: &IndexedDocument<'_>, plan: &ProjectPlan) -> Result<Vec<u8>, Error> {
let mut out = Vec::new();
project_to_sink_indexed(doc, plan, &mut out)?;
Ok(out)
}
pub fn project_indexed_prepare(
doc: &mut IndexedDocument<'_>,
plan: &ProjectPlan,
) -> Result<Vec<u8>, Error> {
doc.index_for_plan(plan)?;
project_indexed(doc, plan)
}
pub fn project_auto_indexed(json: &[u8], plan: &ProjectPlan) -> Result<Vec<u8>, Error> {
let mut doc = IndexedDocument::empty(json);
doc.index_for_plan(plan)?;
project_indexed(&doc, plan)
}
pub fn project_object_fields(
json: &[u8],
array_path: &str,
fields: &[&str],
) -> Result<Vec<u8>, Error> {
let plan = plan_object_fields(array_path, fields)?;
project(json, &plan)
}
pub fn plan_object_fields(array_path: &str, fields: &[&str]) -> Result<ProjectPlan, Error> {
if fields.is_empty() {
return Err(Error::InvalidPath {
msg: "project_object_fields requires at least one field name",
});
}
let multi = fields
.iter()
.map(|f| format!("{f}: {f}"))
.collect::<Vec<_>>()
.join(", ");
let expr = if array_path.is_empty() || array_path == "@" {
format!("[*].{{{multi}}}")
} else {
format!("{array_path}[*].{{{multi}}}")
};
ProjectPlan::from_jmespath(&expr)
}
#[cfg(feature = "parallel")]
pub fn project_parallel(json: &[u8], plan: &ProjectPlan) -> Result<Vec<u8>, Error> {
let mut doc = IndexedDocument::empty(json);
doc.index_for_plan(plan)?;
project_indexed_parallel(&doc, plan)
}
#[cfg(feature = "parallel")]
pub fn project_indexed_parallel(
doc: &IndexedDocument<'_>,
plan: &ProjectPlan,
) -> Result<Vec<u8>, Error> {
let mut out = Vec::new();
let start = skip_whitespace(doc.as_bytes(), 0);
if start >= doc.as_bytes().len() {
return Err(Error::InvalidJsonSyntax {
pos: start,
msg: "Unexpected EOF",
});
}
let end = match plan.select() {
SelectExpr::Identity | SelectExpr::Current => skip_value(doc.as_bytes(), start)?,
_ => {
let mut end = doc.as_bytes().len();
while end > start && matches!(doc.as_bytes()[end - 1], b' ' | b'\t' | b'\n' | b'\r') {
end -= 1;
}
end
}
};
let mut ctx = emit::EmitCtx::new(plan, Some(doc));
#[cfg(feature = "parallel")]
{
ctx.allow_parallel = true;
}
emit::emit_value(doc.as_bytes(), start, end, plan.select(), &mut ctx, &mut out)?;
Ok(out)
}
#[cfg(feature = "parallel")]
pub fn project_object_fields_parallel(
json: &[u8],
array_path: &str,
fields: &[&str],
) -> Result<Vec<u8>, Error> {
let plan = plan_object_fields(array_path, fields)?;
project_parallel(json, &plan)
}
pub fn project_each<F>(json: &[u8], plan: &ProjectPlan, f: F) -> Result<(), Error>
where
F: FnMut(usize, &[u8]) -> Result<(), Error>,
{
project_each_inner(json, plan, None, f)
}
pub fn project_each_indexed<F>(
doc: &IndexedDocument<'_>,
plan: &ProjectPlan,
f: F,
) -> Result<(), Error>
where
F: FnMut(usize, &[u8]) -> Result<(), Error>,
{
project_each_inner(doc.as_bytes(), plan, Some(doc), f)
}
pub fn project_object_fields_each<F>(
json: &[u8],
array_path: &str,
fields: &[&str],
f: F,
) -> Result<(), Error>
where
F: FnMut(usize, &[u8]) -> Result<(), Error>,
{
let plan = plan_object_fields(array_path, fields)?;
project_each(json, &plan, f)
}
pub fn project_object_fields_each_indexed<F>(
doc: &IndexedDocument<'_>,
array_path: &str,
fields: &[&str],
f: F,
) -> Result<(), Error>
where
F: FnMut(usize, &[u8]) -> Result<(), Error>,
{
let plan = plan_object_fields(array_path, fields)?;
project_each_indexed(doc, &plan, f)
}
pub fn project_jsonl_write<W: Write>(
json: &[u8],
plan: &ProjectPlan,
mut w: W,
) -> Result<usize, Error> {
let mut written = 0usize;
project_each(json, plan, |_, card| {
write_card_line(&mut w, card, &mut written)
})?;
Ok(written)
}
pub fn project_object_fields_jsonl_write<W: Write>(
json: &[u8],
array_path: &str,
fields: &[&str],
w: W,
) -> Result<usize, Error> {
let plan = plan_object_fields(array_path, fields)?;
project_jsonl_write(json, &plan, w)
}
pub fn project_jsonl_write_indexed<W: Write>(
doc: &IndexedDocument<'_>,
plan: &ProjectPlan,
mut w: W,
) -> Result<usize, Error> {
let mut written = 0usize;
project_each_indexed(doc, plan, |_, card| {
write_card_line(&mut w, card, &mut written)
})?;
Ok(written)
}
fn write_card_line<W: Write>(w: &mut W, card: &[u8], written: &mut usize) -> Result<(), Error> {
w.write_all(card).map_err(|_| Error::InvalidJsonSyntax {
pos: 0,
msg: "Write error while emitting JSONL card",
})?;
w.write_all(b"\n").map_err(|_| Error::InvalidJsonSyntax {
pos: 0,
msg: "Write error while emitting JSONL newline",
})?;
*written = written.saturating_add(card.len()).saturating_add(1);
Ok(())
}
fn project_each_inner<F>(
json: &[u8],
plan: &ProjectPlan,
index: Option<&IndexedDocument<'_>>,
f: F,
) -> Result<(), Error>
where
F: FnMut(usize, &[u8]) -> Result<(), Error>,
{
let (array_path, list_sel) = peel_list_proj(plan.select()).ok_or(Error::Jmespath {
msg: "project_each requires a list projection plan (e.g. items[*].{…}, items[?…], items[0:n], or [*].…)",
})?;
let each = list_each_expr(list_sel).ok_or(Error::Jmespath {
msg: "project_each does not support multi-index list projections",
})?;
let (open, end) = resolve_array_open_end(json, &array_path, index)?;
let mut ctx = emit::EmitCtx::new(plan, index);
emit::for_each_projected_card(json, open, end, list_sel, each, &mut ctx, f)
}
fn resolve_array_open_end(
json: &[u8],
array_path: &str,
index: Option<&IndexedDocument<'_>>,
) -> Result<(usize, usize), Error> {
let (start, end) = if array_path.is_empty() {
let start = skip_whitespace(json, 0);
let end = skip_value(json, start)?;
(start, end)
} else {
let segs = parse_path(array_path);
if let Some(doc) = index {
match doc.find_offsets(&segs) {
Ok(span) => span,
Err(_) => find_value_offsets(json, &segs)?,
}
} else {
find_value_offsets(json, &segs)?
}
};
let open = skip_whitespace(json, start);
if open >= json.len() || json[open] != b'[' {
return Err(Error::TypeMismatch {
expected: "array",
found: match json.get(open).copied() {
Some(b'{') => "object",
Some(b'"') => "string",
Some(b't') | Some(b'f') => "bool",
Some(b'n') => "null",
Some(b'-') | Some(b'0'..=b'9') => "number",
_ => "unknown",
},
});
}
Ok((open, end))
}
fn peel_list_proj(expr: &SelectExpr) -> Option<(String, &ArraySelect)> {
match expr {
SelectExpr::Paren(inner) => peel_list_proj(inner),
SelectExpr::Array(sel) if is_streamable_list(sel) => Some((String::new(), sel)),
SelectExpr::Sub(left, right) => match right.as_ref() {
SelectExpr::Array(sel) if is_streamable_list(sel) => {
let path = field_chain_path(left)?;
Some((path, sel))
}
SelectExpr::Sub(_, _) | SelectExpr::Paren(_) | SelectExpr::Array(_) => {
let (suffix, sel) = peel_list_proj(right)?;
let prefix = field_chain_path(left)?;
let full = join_dot_path(&prefix, &suffix);
Some((full, sel))
}
_ => None,
},
_ => None,
}
}
fn is_streamable_list(sel: &ArraySelect) -> bool {
matches!(
sel,
ArraySelect::Each(_) | ArraySelect::Filter { .. } | ArraySelect::Slice { .. }
)
}
fn list_each_expr(sel: &ArraySelect) -> Option<&SelectExpr> {
match sel {
ArraySelect::Each(each)
| ArraySelect::Filter { each, .. }
| ArraySelect::Slice { each, .. } => Some(each.as_ref()),
ArraySelect::Indices(_) => None,
}
}
fn field_chain_path(expr: &SelectExpr) -> Option<String> {
match expr {
SelectExpr::Identity | SelectExpr::Current => Some(String::new()),
SelectExpr::Field(k) | SelectExpr::FieldQuoted(k) => Some(k.clone()),
SelectExpr::Paren(inner) => field_chain_path(inner),
SelectExpr::Sub(left, right) => {
let a = field_chain_path(left)?;
let b = field_chain_path(right)?;
Some(join_dot_path(&a, &b))
}
_ => None,
}
}
fn join_dot_path(prefix: &str, suffix: &str) -> String {
if prefix.is_empty() {
suffix.to_string()
} else if suffix.is_empty() {
prefix.to_string()
} else {
format!("{prefix}.{suffix}")
}
}
pub fn plan_prefers_parallel(plan: &ProjectPlan) -> bool {
expr_prefers_parallel(plan.select())
}
fn expr_prefers_parallel(expr: &SelectExpr) -> bool {
match expr {
SelectExpr::Call { .. } => true,
SelectExpr::Array(ArraySelect::Filter { .. }) => true,
SelectExpr::Array(ArraySelect::Each(c)) => expr_prefers_parallel(c),
SelectExpr::Array(ArraySelect::Slice { each, .. }) => expr_prefers_parallel(each),
SelectExpr::Array(ArraySelect::Indices(map)) => map.values().any(expr_prefers_parallel),
SelectExpr::MultiSelectHash(fields) => fields.iter().any(|f| match &f.expr {
SelectExpr::Field(_) | SelectExpr::FieldQuoted(_) | SelectExpr::Identity
| SelectExpr::Current | SelectExpr::Literal(_) => false,
other => expr_prefers_parallel(other),
}),
SelectExpr::MultiSelectList(items) => items.iter().any(expr_prefers_parallel),
SelectExpr::Sub(l, r) | SelectExpr::Pipe(l, r) | SelectExpr::And(l, r)
| SelectExpr::Or(l, r) => expr_prefers_parallel(l) || expr_prefers_parallel(r),
SelectExpr::Cmp { left, right, .. } => {
expr_prefers_parallel(left) || expr_prefers_parallel(right)
}
SelectExpr::Flatten(i)
| SelectExpr::Not(i)
| SelectExpr::Paren(i)
| SelectExpr::Expref(i)
| SelectExpr::ObjectProjection(i) => expr_prefers_parallel(i),
SelectExpr::Object(obj) => obj.fields.values().any(expr_prefers_parallel),
SelectExpr::Identity
| SelectExpr::Current
| SelectExpr::Field(_)
| SelectExpr::FieldQuoted(_)
| SelectExpr::Literal(_) => false,
}
}
pub fn project_indexed_auto(
doc: &IndexedDocument<'_>,
plan: &ProjectPlan,
) -> Result<Vec<u8>, Error> {
#[cfg(feature = "parallel")]
if plan_prefers_parallel(plan) {
return project_indexed_parallel(doc, plan);
}
project_indexed(doc, plan)
}
pub fn project_parallel_auto(json: &[u8], plan: &ProjectPlan) -> Result<Vec<u8>, Error> {
let mut doc = IndexedDocument::empty(json);
doc.index_for_plan(plan)?;
project_indexed_auto(&doc, plan)
}
pub fn project_write<W: Write>(json: &[u8], plan: &ProjectPlan, w: W) -> Result<usize, Error> {
let mut sink = WSink::new(w);
project_to_sink(json, plan, &mut sink)?;
Ok(sink.written)
}
pub fn projected_len(json: &[u8], plan: &ProjectPlan) -> Result<usize, Error> {
let mut c = CountSink::default();
project_to_sink(json, plan, &mut c)?;
Ok(c.len)
}
fn project_to_sink(json: &[u8], plan: &ProjectPlan, out: &mut impl EmitOut) -> Result<(), Error> {
project_to_sink_inner(json, plan, None, out)
}
fn project_to_sink_indexed(
doc: &IndexedDocument<'_>,
plan: &ProjectPlan,
out: &mut impl EmitOut,
) -> Result<(), Error> {
project_to_sink_inner(doc.as_bytes(), plan, Some(doc), out)
}
fn project_to_sink_inner(
json: &[u8],
plan: &ProjectPlan,
index: Option<&IndexedDocument<'_>>,
out: &mut impl EmitOut,
) -> Result<(), Error> {
let start = skip_whitespace(json, 0);
if start >= json.len() {
return Err(Error::InvalidJsonSyntax {
pos: start,
msg: "Unexpected EOF",
});
}
let end = match plan.select() {
crate::project::select::SelectExpr::Identity
| crate::project::select::SelectExpr::Current => skip_value(json, start)?,
_ => {
let mut end = json.len();
while end > start && matches!(json[end - 1], b' ' | b'\t' | b'\n' | b'\r') {
end -= 1;
}
end
}
};
let mut ctx = emit::EmitCtx::new(plan, index);
emit::emit_value(json, start, end, plan.select(), &mut ctx, out)
}
pub fn project_paths(json: &[u8], paths: &[&str]) -> Result<Vec<u8>, Error> {
project(json, &ProjectPlan::from_paths(paths)?)
}
pub fn project_jmespath(json: &[u8], expr: &str) -> Result<Vec<u8>, Error> {
project(json, &ProjectPlan::from_jmespath(expr)?)
}
pub fn estimate_projected_len(json: &[u8], paths: &[&str]) -> Result<usize, Error> {
if paths.is_empty() {
return Ok(2);
}
let mut total = 1usize;
for (i, p) in paths.iter().enumerate() {
if i > 0 {
total += 1;
}
let segs = parse_path(p);
let val = find_value(json, &segs)?;
total += key_wire_len(&segs);
total += 1;
total += val.len();
}
total += 1;
Ok(total)
}
pub fn estimate_values_len(json: &[u8], paths: &[&str]) -> Result<usize, Error> {
let mut total = 0usize;
for p in paths {
total += find_value(json, &parse_path(p))?.len();
}
Ok(total)
}
fn key_wire_len(segs: &[PathSegment<'_>]) -> usize {
for seg in segs.iter().rev() {
if let PathSegment::Key(k) = seg {
return 2 + k.len();
}
}
2
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn project_top_level_keys() {
let json = br#"{"id":1,"title":"x","blob":true}"#;
assert_eq!(
project_paths(json, &["id", "title"]).unwrap(),
br#"{"id":1,"title":"x"}"#
);
}
#[test]
fn project_jmespath_multi_hash_rename() {
let json = br#"{"products":[{"id":1,"t":"a"},{"id":2,"t":"b"}]}"#;
let out = project_jmespath(json, "products[*].{id: id, title: t}").unwrap();
assert_eq!(
out,
br#"[{"id":1,"title":"a"},{"id":2,"title":"b"}]"#
);
}
#[test]
fn project_jmespath_slice() {
let json = br#"{"a":[10,20,30,40]}"#;
let out = project_jmespath(json, "a[1:3]").unwrap();
assert_eq!(out, br#"[20,30]"#);
}
#[test]
fn project_jmespath_negative_index() {
let json = br#"{"a":[10,20,30]}"#;
assert_eq!(project_jmespath(json, "a[-1]").unwrap(), b"30");
assert_eq!(project_jmespath(json, "a[-2]").unwrap(), b"20");
}
#[test]
fn project_jmespath_filter() {
let json = br#"{"items":[{"n":1,"ok":true},{"n":2,"ok":false},{"n":3,"ok":true}]}"#;
let out = project_jmespath(json, "items[?ok == `true`].n").unwrap();
assert_eq!(out, br#"[1,3]"#);
}
#[test]
fn project_jmespath_functions() {
let json = br#"{"a":[3,1,2],"t":"Hello"}"#;
assert_eq!(project_jmespath(json, "length(a)").unwrap(), b"3");
assert_eq!(project_jmespath(json, "length(t)").unwrap(), b"5");
assert_eq!(project_jmespath(json, "sort(a)").unwrap(), br#"[1,2,3]"#);
assert_eq!(project_jmespath(json, "max(a)").unwrap(), b"3");
assert_eq!(
project_jmespath(json, "starts_with(t, 'He')").unwrap(),
b"true"
);
assert_eq!(project_jmespath(json, "type(a)").unwrap(), br#""array""#);
}
#[test]
fn project_jmespath_paren_and_or() {
let json = br#"{"x":1,"y":0}"#;
assert_eq!(
project_jmespath(json, "(x == `1`) && (y == `0`)").unwrap(),
b"true"
);
assert_eq!(project_jmespath(json, "x == `2` || y == `0`").unwrap(), b"true");
assert_eq!(project_jmespath(json, "!(x == `2`)").unwrap(), b"true");
}
#[test]
fn project_paths_keeps_nesting_wrapper() {
let json = br#"{"products":[{"id":1,"t":"a"},{"id":2,"t":"b"}]}"#;
let out = project_paths(json, &["products[].id", "products[].t"]).unwrap();
assert_eq!(
out,
br#"{"products":[{"id":1,"t":"a"},{"id":2,"t":"b"}]}"#
);
}
#[test]
fn project_jmespath_pipe_flatten() {
let json = br#"{"x":[[1,2],[3],[4,5]]}"#;
let out = project_jmespath(json, "x | []").unwrap();
assert_eq!(out, br#"[1,2,3,4,5]"#);
}
#[test]
fn project_jmespath_multi_list() {
let json = br#"{"id":1,"title":"h"}"#;
let out = project_jmespath(json, "[id, title]").unwrap();
assert_eq!(out, br#"[1,"h"]"#);
}
#[test]
fn project_jmespath_literal_field() {
let json = br#"{"id":1}"#;
let out = project_jmespath(json, "{id: id, source: 'teefury'}").unwrap();
assert_eq!(out, br#"{"id":1,"source":"teefury"}"#);
}
#[test]
fn project_array_wildcard_paths() {
let json = br#"{"products":[{"id":1,"t":"a"},{"id":2,"t":"b"}]}"#;
let out = project_paths(json, &["products[].id"]).unwrap();
assert_eq!(out, br#"{"products":[{"id":1},{"id":2}]}"#);
}
#[test]
fn project_pretty() {
let plan = ProjectPlan::from_paths(&["a", "b"])
.unwrap()
.style(ProjectStyle::Pretty { indent: 2 });
let out = project(br#"{"a":1,"b":2}"#, &plan).unwrap();
assert_eq!(
std::str::from_utf8(&out).unwrap(),
"{\n \"a\": 1,\n \"b\": 2\n}"
);
}
#[test]
fn project_preserve_source() {
let json = br#"{ "id" : 1 , "drop": 2 }"#;
let plan = ProjectPlan::from_paths(&["id"])
.unwrap()
.style(ProjectStyle::PreserveSource);
assert_eq!(project(json, &plan).unwrap(), br#"{ "id" : 1 }"#);
}
#[test]
fn project_nested_merge() {
let json = br#"{"user":{"name":"a","age":9,"city":"z"},"x":0}"#;
assert_eq!(
project_paths(json, &["user.name", "user.age"]).unwrap(),
br#"{"user":{"name":"a","age":9}}"#
);
}
#[test]
fn project_write_streams_and_counts() {
let json = br#"{"id":1,"title":"x","blob":[1,2,3]}"#;
let plan = ProjectPlan::from_paths(&["id", "title"]).unwrap();
let mut buf = Vec::new();
let n = project_write(json, &plan, &mut buf).unwrap();
assert_eq!(n, buf.len());
assert_eq!(buf, br#"{"id":1,"title":"x"}"#);
assert_eq!(projected_len(json, &plan).unwrap(), n);
}
#[test]
fn project_indexed_matches_project() {
let json = br#"{"products":[{"id":1},{"id":2}]}"#;
let plan = ProjectPlan::from_paths(&["products[].id"]).unwrap();
let doc = crate::IndexedDocument::build(json, &["products"]).unwrap();
assert_eq!(
project_indexed(&doc, &plan).unwrap(),
project(json, &plan).unwrap()
);
}
#[test]
fn flatten_projection_vs_list_projection() {
let json = br#"{"reservations":[{"instances":[{"foo":1},{"foo":2}]}]}"#;
assert_eq!(
project_jmespath(json, "reservations[].instances[].foo").unwrap(),
br#"[1,2]"#
);
assert_eq!(
project_jmespath(json, "reservations[*].instances[*].foo").unwrap(),
br#"[[1,2]]"#
);
}
#[test]
fn pure_pipe_matches_direct_path() {
let json = br#"{"a":{"b":{"c":42}}}"#;
assert_eq!(
project_jmespath(json, "a.b | c").unwrap(),
project_jmespath(json, "a.b.c").unwrap()
);
}
#[test]
fn plan_array_paths_for_index() {
let plan = ProjectPlan::from_paths(&["products[0].id", "products[2].title"]).unwrap();
assert_eq!(plan.array_paths_for_index(), vec!["products".to_string()]);
let plan = ProjectPlan::from_jmespath("products[*].{id: id, title: title}").unwrap();
assert_eq!(plan.array_paths_for_index(), vec!["products".to_string()]);
}
#[test]
fn multi_select_pure_fields_one_pass_semantics() {
let json = br#"{"z":9,"a":1,"noise":true,"b":"two","c":{"nested":3}}"#;
assert_eq!(
project_jmespath(json, "{first: a, alias: b, last: z}").unwrap(),
br#"{"first":1,"alias":"two","last":9}"#
);
assert_eq!(
project_jmespath(json, "{a: a, missing: nope, b: b}").unwrap(),
br#"{"a":1,"missing":null,"b":"two"}"#
);
assert_eq!(
project_jmespath(json, "{x: a, y: a}").unwrap(),
br#"{"x":1,"y":1}"#
);
assert_eq!(
project_jmespath(json, "{a: a, n: c.nested}").unwrap(),
br#"{"a":1,"n":3}"#
);
let arr = br#"[{"id":1,"t":"a","x":9},{"id":2,"t":"b"},{"id":3,"t":"c","x":1}]"#;
assert_eq!(
project_jmespath(arr, "[*].{id: id, t: t, x: x}").unwrap(),
br#"[{"id":1,"t":"a","x":9},{"id":2,"t":"b","x":null},{"id":3,"t":"c","x":1}]"#
);
}
#[test]
fn project_object_fields_generic_shapes() {
let root = br#"[{"a":1,"b":2,"c":3},{"a":4,"b":5,"c":6}]"#;
assert_eq!(
project_object_fields(root, "", &["a", "c"]).unwrap(),
br#"[{"a":1,"c":3},{"a":4,"c":6}]"#
);
let nested = br#"{"meta":{"rows":[{"x":9,"y":8,"z":7}]}}"#;
assert_eq!(
project_object_fields(nested, "meta.rows", &["z", "x"]).unwrap(),
br#"[{"z":7,"x":9}]"#
);
}
#[test]
fn project_each_and_jsonl_match_array_project() {
let json = br#"{"items":[{"id":1,"t":"a","x":9},{"id":2,"t":"b"},{"id":3,"t":"c","x":1}]}"#;
let plan = ProjectPlan::from_jmespath("items[*].{id: id, t: t}").unwrap();
let as_array = project(json, &plan).unwrap();
let mut cards = Vec::new();
project_each(json, &plan, |i, card| {
cards.push((i, card.to_vec()));
Ok(())
})
.unwrap();
assert_eq!(cards.len(), 3);
assert_eq!(cards[0].1, br#"{"id":1,"t":"a"}"#);
assert_eq!(cards[2].1, br#"{"id":3,"t":"c"}"#);
let mut rebuilt = Vec::from(b"[".as_slice());
for (i, (_, c)) in cards.iter().enumerate() {
if i > 0 {
rebuilt.push(b',');
}
rebuilt.extend_from_slice(c);
}
rebuilt.push(b']');
assert_eq!(rebuilt, as_array);
let mut jsonl = Vec::new();
let n = project_jsonl_write(json, &plan, &mut jsonl).unwrap();
assert_eq!(n, jsonl.len());
assert_eq!(
&jsonl[..],
b"{\"id\":1,\"t\":\"a\"}\n{\"id\":2,\"t\":\"b\"}\n{\"id\":3,\"t\":\"c\"}\n"
);
let nested = br#"{"meta":{"rows":[{"z":7,"x":9,"y":1}]}}"#;
let mut got = None;
project_object_fields_each(nested, "meta.rows", &["z", "x"], |_, card| {
got = Some(card.to_vec());
Ok(())
})
.unwrap();
assert_eq!(got.unwrap(), br#"{"z":7,"x":9}"#);
let root = br#"[{"a":1,"b":2},{"a":3,"b":4}]"#;
let mut jsonl = Vec::new();
project_object_fields_jsonl_write(root, "", &["a"], &mut jsonl).unwrap();
assert_eq!(&jsonl[..], b"{\"a\":1}\n{\"a\":3}\n");
let mut doc = crate::IndexedDocument::empty(json);
doc.index_for_plan(&plan).unwrap();
let mut n = 0usize;
project_each_indexed(&doc, &plan, |_, _| {
n += 1;
Ok(())
})
.unwrap();
assert_eq!(n, 3);
}
#[test]
fn project_each_filter_and_slice_match_array_project() {
let json = br#"{"items":[
{"id":1,"ok":true,"t":"a"},
{"id":2,"ok":false,"t":"b"},
{"id":3,"ok":true,"t":"c"},
{"id":4,"ok":true,"t":"d"}
]}"#;
let plan_f = ProjectPlan::from_jmespath("items[?ok].{id: id, t: t}").unwrap();
let as_array = project(json, &plan_f).unwrap();
let mut cards = Vec::new();
project_each(json, &plan_f, |i, card| {
cards.push((i, card.to_vec()));
Ok(())
})
.unwrap();
assert_eq!(cards.len(), 3);
assert_eq!(cards[0].1, br#"{"id":1,"t":"a"}"#);
assert_eq!(cards[1].1, br#"{"id":3,"t":"c"}"#);
assert_eq!(cards[2].1, br#"{"id":4,"t":"d"}"#);
let mut rebuilt = Vec::from(b"[".as_slice());
for (i, (_, c)) in cards.iter().enumerate() {
if i > 0 {
rebuilt.push(b',');
}
rebuilt.extend_from_slice(c);
}
rebuilt.push(b']');
assert_eq!(rebuilt, as_array);
let plan_s = ProjectPlan::from_jmespath("items[1:3].{id: id}").unwrap();
let as_array = project(json, &plan_s).unwrap();
let mut cards = Vec::new();
project_each(json, &plan_s, |_, card| {
cards.push(card.to_vec());
Ok(())
})
.unwrap();
assert_eq!(cards, vec![br#"{"id":2}"#.to_vec(), br#"{"id":3}"#.to_vec()]);
let mut rebuilt = Vec::from(b"[".as_slice());
for (i, c) in cards.iter().enumerate() {
if i > 0 {
rebuilt.push(b',');
}
rebuilt.extend_from_slice(c);
}
rebuilt.push(b']');
assert_eq!(rebuilt, as_array);
let root = br#"[10,20,30,40]"#;
let plan_r = ProjectPlan::from_jmespath("[1:3]").unwrap();
let mut got = Vec::new();
project_each(root, &plan_r, |_, card| {
got.push(card.to_vec());
Ok(())
})
.unwrap();
assert_eq!(got, vec![b"20".to_vec(), b"30".to_vec()]);
assert_eq!(project(root, &plan_r).unwrap(), br#"[20,30]"#);
let mut jsonl = Vec::new();
project_jsonl_write(json, &plan_f, &mut jsonl).unwrap();
assert_eq!(
&jsonl[..],
b"{\"id\":1,\"t\":\"a\"}\n{\"id\":3,\"t\":\"c\"}\n{\"id\":4,\"t\":\"d\"}\n"
);
}
#[test]
fn plan_prefers_parallel_heuristic() {
let thin = ProjectPlan::from_jmespath("items[*].{id: id, title: title}").unwrap();
assert!(
!plan_prefers_parallel(&thin),
"pure-field multi-select should stay sequential"
);
let heavy = ProjectPlan::from_jmespath(
"records[*].{id: id, n: length(scores[?@ > `0.5`])}",
)
.unwrap();
assert!(
plan_prefers_parallel(&heavy),
"filter + length should prefer parallel"
);
let filtered = ProjectPlan::from_jmespath("items[?id > `0`].id").unwrap();
assert!(plan_prefers_parallel(&filtered));
}
#[test]
fn project_indexed_auto_thin_matches_sequential() {
let mut json = String::from(r#"{"rows":["#);
for i in 0..80 {
if i > 0 {
json.push(',');
}
json.push_str(&format!(r#"{{"id":{i},"name":"n{i}"}}"#));
}
json.push_str("]}");
let bytes = json.as_bytes();
let plan = ProjectPlan::from_jmespath("rows[*].{id: id, name: name}").unwrap();
let mut doc = crate::IndexedDocument::empty(bytes);
doc.index_for_plan(&plan).unwrap();
let seq = project_indexed(&doc, &plan).unwrap();
let auto = project_indexed_auto(&doc, &plan).unwrap();
assert_eq!(seq, auto);
let prep = project_parallel_auto(bytes, &plan).unwrap();
assert_eq!(seq, prep);
}
#[cfg(feature = "parallel")]
#[test]
fn project_parallel_matches_sequential_on_array() {
let mut json = String::from(r#"{"rows":["#);
for i in 0..200 {
if i > 0 {
json.push(',');
}
json.push_str(&format!(r#"{{"id":{i},"name":"n{i}","noise":true}}"#));
}
json.push_str("]}");
let bytes = json.as_bytes();
let seq = project_object_fields(bytes, "rows", &["id", "name"]).unwrap();
let par = project_object_fields_parallel(bytes, "rows", &["id", "name"]).unwrap();
assert_eq!(seq, par);
let plan = ProjectPlan::from_jmespath("rows[*].{id: id, name: name}").unwrap();
let mut doc = crate::IndexedDocument::empty(bytes);
doc.index_for_plan(&plan).unwrap();
assert_eq!(project_indexed_parallel(&doc, &plan).unwrap(), seq);
}
#[test]
fn project_auto_indexed_mid_element() {
let mut json = String::from(r#"{"products":["#);
for i in 0..100 {
if i > 0 {
json.push(',');
}
json.push_str(&format!(r#"{{"id":{i},"title":"T{i}"}}"#));
}
json.push_str("]}");
let bytes = json.as_bytes();
let plan = ProjectPlan::from_paths(&["products[50].id"]).unwrap();
let out = project_auto_indexed(bytes, &plan).unwrap();
assert_eq!(out, br#"{"products":{"id":50}}"#);
let mut doc = crate::IndexedDocument::empty(bytes);
let out2 = project_indexed_prepare(&mut doc, &plan).unwrap();
assert_eq!(out2, out);
assert!(doc.array_len(&crate::parse_path("products")).unwrap() >= 100);
}
#[test]
fn single_index_short_circuit_correctness() {
let mut json = String::from(r#"{"products":["#);
for i in 0..200 {
if i > 0 {
json.push(',');
}
json.push_str(&format!(
r#"{{"id":{i},"title":"P{i}","pad":"{pad}"}}"#,
pad = "x".repeat(64)
));
}
json.push_str("]}");
let bytes = json.as_bytes();
assert_eq!(project_jmespath(bytes, "products[0].id").unwrap(), b"0");
assert_eq!(project_jmespath(bytes, "products[99].id").unwrap(), b"99");
assert_eq!(project_jmespath(bytes, "products[199].id").unwrap(), b"199");
assert_eq!(project_jmespath(bytes, "products[-1].id").unwrap(), b"199");
assert_eq!(
project_jmespath(bytes, "products[-1].title").unwrap(),
br#""P199""#
);
let multi =
project_jmespath(bytes, "[products[0].id, products[2].id, products[-1].id]").unwrap();
let v: serde_json::Value = serde_json::from_slice(&multi).unwrap();
assert_eq!(v, serde_json::json!([0, 2, 199]));
let mut doc = crate::IndexedDocument::empty(bytes);
doc.index_array_str("products").unwrap();
let plan = ProjectPlan::from_paths(&["products[0].id", "products[0].title"]).unwrap();
let out = project_indexed(&doc, &plan).unwrap();
assert_eq!(out, project(bytes, &plan).unwrap());
assert_eq!(out, br#"{"products":{"id":0,"title":"P0"}}"#);
}
#[test]
fn p0_synthetic_sparse_index_is_fast() {
let mut json = String::from(r#"{"products":["#);
let pad = "x".repeat(256);
for i in 0..50_000 {
if i > 0 { json.push(','); }
json.push_str(&format!(r#"{{"id":{i},"title":"T{i}","pad":"{pad}"}}"#));
}
json.push_str("]}");
let bytes = json.as_bytes();
let t0 = std::time::Instant::now();
let out = project_jmespath(bytes, "products[0].id").unwrap();
let dt = t0.elapsed();
assert_eq!(out, b"0");
assert!(
dt.as_millis() < 25,
"products[0].id took {dt:?} — sparse short-circuit not active?"
);
let t1 = std::time::Instant::now();
let out = project_paths(bytes, &["products[0].id", "products[0].title"]).unwrap();
let dt1 = t1.elapsed();
assert_eq!(out, br#"{"products":{"id":0,"title":"T0"}}"#);
assert!(
dt1.as_millis() < 25,
"keep-list products[0] took {dt1:?}"
);
let mut doc = crate::IndexedDocument::empty(bytes);
doc.index_array_str("products").unwrap();
let plan = ProjectPlan::from_paths(&["products[24999].id"]).unwrap();
let t2 = std::time::Instant::now();
let out = project_indexed(&doc, &plan).unwrap();
let dt2 = t2.elapsed();
assert_eq!(out, br#"{"products":{"id":24999}}"#);
assert!(
dt2.as_millis() < 15,
"indexed products[24999] took {dt2:?}"
);
}
#[test]
fn transform_pipeline_rename_drop_inject() {
use crate::project::transform::{Transform, TransformPipeline};
let json = br#"{"id":1,"title":"Hat","noise":true}"#;
let out = TransformPipeline::new()
.then(Transform::KeepPaths(&["id", "title"]))
.then(Transform::Rename {
from: "title",
to: "name",
})
.then(Transform::Inject {
key: "source",
value: br#""api""#,
})
.then(Transform::Drop("id"))
.apply(json)
.unwrap();
let v: serde_json::Value = serde_json::from_slice(&out).unwrap();
assert_eq!(v["name"], "Hat");
assert_eq!(v["source"], "api");
assert!(v.get("id").is_none());
assert!(v.get("noise").is_none());
}
#[test]
fn preserve_source_array_spacing() {
let json = br#"[ 1 , 2 , 3 ]"#;
let plan = ProjectPlan::from_jmespath("@")
.unwrap()
.style(ProjectStyle::PreserveSource);
let out = project(json, &plan).unwrap();
assert_eq!(out, json);
}
#[test]
fn serde_accepts_projection() {
let json = br#"{"id":7,"title":"Hat","images":[1,2,3],"meta":{"x":1}}"#;
let out = project_paths(json, &["id", "title", "meta.x"]).unwrap();
let v: serde_json::Value = serde_json::from_slice(&out).unwrap();
assert_eq!(v["id"], 7);
assert!(v.get("images").is_none());
}
}