#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct JsonTally {
pub objects: u64,
pub members: u64,
pub key_bytes: u64,
pub arrays: u64,
pub elements: u64,
pub strings: u64,
pub string_bytes: u64,
pub numbers: u64,
pub literals: u64,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Compacted {
pub tally: JsonTally,
pub exact: bool,
pub null: bool,
}
pub fn compact_json(src: &[u8], out: &mut Vec<u8>) -> Compacted {
compact(src, out, None)
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
struct Nulls {
keyed: Vec<std::ops::Range<usize>>,
unkeyed: bool,
}
impl Nulls {
fn note(&mut self, out: &[u8], key: Option<std::ops::Range<usize>>) {
match key {
Some(key) => {
if !self.keyed.iter().any(|seen| out[seen.clone()] == out[key.clone()]) {
self.keyed.push(key);
}
},
None => self.unkeyed = true,
}
}
}
fn compact(src: &[u8], out: &mut Vec<u8>, mut nulls: Option<&mut Nulls>) -> Compacted {
let mut found = Compacted {
exact: true,
..Compacted::default()
};
let mut open: Vec<bool> = Vec::new();
let mut number_start = None;
out.reserve(src.len());
let mut i = 0;
while let Some(&b) = src.get(i) {
if let Some((start, out_start)) = number_start {
if matches!(b, b'0'..=b'9' | b'-' | b'+' | b'.' | b'e' | b'E') {
out.push(b);
i += 1;
continue;
}
found.exact &= write_number(&src[start..i], out, out_start);
number_start = None;
}
let tally = &mut found.tally;
match b {
b' ' | b'\n' | b'\r' | b'\t' => {
i += 1;
continue;
},
b'"' => {
let key = open.last() == Some(&true) && expects_key(out);
if key {
tally.members += 1;
} else {
count_element(&open, tally);
}
out.push(b'"');
let (end, len) = copy_string(src, i + 1, out, &mut found.exact);
let tally = &mut found.tally;
if key {
tally.key_bytes += len;
} else {
tally.strings += 1;
tally.string_bytes += len;
}
i = end;
continue;
},
b'{' | b'[' => {
count_element(&open, tally);
if b == b'{' {
tally.objects += 1;
} else {
tally.arrays += 1;
}
open.push(b == b'{');
},
b'}' | b']' => {
open.pop();
},
b'n' | b't' | b'f' => {
count_element(&open, tally);
tally.literals += 1;
if b == b'n' {
found.null = true;
if let Some(nulls) = nulls.as_deref_mut() {
let key = (open.last() == Some(&true)).then(|| member_key(out)).flatten();
nulls.note(out, key);
}
}
},
b'0'..=b'9' | b'-' => {
count_element(&open, tally);
tally.numbers += 1;
number_start = Some((i, out.len()));
},
_ => {},
}
out.push(b);
i += 1;
}
if let Some((start, out_start)) = number_start {
found.exact &= write_number(&src[start..], out, out_start);
}
found
}
fn member_key(out: &[u8]) -> Option<std::ops::Range<usize>> {
let close = out.len().checked_sub(2)?;
let open = memchr::memrchr(b'"', &out[..close])?;
matches!(open.checked_sub(1).map(|before| out[before]), Some(b'{' | b','))
.then_some(open + 1..close)
}
fn copy_string(src: &[u8], from: usize, out: &mut Vec<u8>, exact: &mut bool) -> (usize, u64) {
let mut i = from;
let mut len = 0u64;
while let Some(run) = src.get(i..) {
let Some(at) = memchr::memchr2(b'"', b'\\', run) else {
out.extend_from_slice(run);
return (src.len(), len + run.len() as u64);
};
out.extend_from_slice(&run[..=at]);
len += at as u64;
i += at + 1;
if run[at] == b'"' {
return (i, len);
}
if let Some(&escaped) = src.get(i) {
out.push(escaped);
*exact &= escaped != b'u';
len += 1;
i += 1;
}
}
(src.len(), len)
}
const fn expects_key(out: &[u8]) -> bool {
matches!(out.last(), Some(b'{' | b','))
}
fn count_element(open: &[bool], tally: &mut JsonTally) {
if open.last() == Some(&false) {
tally.elements += 1;
}
}
fn write_number(token: &[u8], out: &mut Vec<u8>, at: usize) -> bool {
if exact_integer(token) {
return true;
}
let Ok(number) = serde_json::from_slice::<serde_json::Number>(token) else {
return false;
};
out.truncate(at);
if serde_json::to_writer(&mut *out, &number).is_ok() {
return true;
}
out.truncate(at);
out.extend_from_slice(token);
false
}
fn exact_integer(token: &[u8]) -> bool {
let digits = token.strip_prefix(b"-").unwrap_or(token);
!digits.is_empty()
&& digits.len() <= 18
&& digits.iter().all(u8::is_ascii_digit)
&& token != b"-0"
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct RawRows {
bytes: Vec<u8>,
rows: Vec<RawRow>,
exact: bool,
nulls: Nulls,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RawRow {
pub end: usize,
pub tally: JsonTally,
}
impl RawRows {
#[must_use]
pub const fn new() -> Self {
Self {
bytes: Vec::new(),
rows: Vec::new(),
exact: true,
nulls: Nulls {
keyed: Vec::new(),
unkeyed: false,
},
}
}
pub fn push(&mut self, src: &[u8]) {
if !self.rows.is_empty() {
self.bytes.push(b',');
}
let found = compact(src, &mut self.bytes, Some(&mut self.nulls));
self.exact &= found.exact;
self.rows.push(RawRow {
end: self.bytes.len(),
tally: found.tally,
});
}
#[must_use]
pub fn bytes(&self) -> &[u8] {
&self.bytes
}
#[must_use]
pub fn rows(&self) -> &[RawRow] {
&self.rows
}
#[must_use]
pub fn row(&self, index: usize) -> Option<&[u8]> {
let row = self.rows.get(index)?;
let start = match index {
0 => 0,
_ => self.rows[index - 1].end + 1,
};
Some(&self.bytes[start..row.end])
}
#[must_use]
pub const fn exact(&self) -> bool {
self.exact
}
#[must_use]
pub fn has_null(&self) -> bool {
self.has_null_where(|_| true)
}
#[must_use]
pub fn has_null_where(&self, at: impl Fn(Option<&[u8]>) -> bool) -> bool {
(self.nulls.unkeyed && at(None))
|| self.nulls.keyed.iter().any(|key| at(Some(&self.bytes[key.clone()])))
}
pub fn parse(&self) -> serde_json::Result<Vec<serde_json::Value>> {
(0..self.rows.len())
.map(|i| serde_json::from_slice(self.row(i).unwrap_or_default()))
.collect()
}
}
#[cfg(test)]
mod tests;