#![warn(missing_docs)]
#![warn(rustdoc::broken_intra_doc_links)]
#![cfg_attr(docsrs, feature(doc_cfg))]
#[cfg(doctest)]
#[doc = include_str!("../README.md")]
mod readme {}
mod config;
mod event;
mod parse;
mod prelude;
mod scan;
mod stream;
pub use config::Config;
pub use event::{AttrIter, Attribute, Attrs, Event};
pub use parse::RecordReader;
pub use prelude::{Encoding, NamespaceContext, Prelude};
pub use scan::ChunkIndex;
pub use stream::StreamReader;
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt;
use std::ops::Range;
use std::path::Path;
use std::sync::Arc;
use quick_xml::Reader;
use quick_xml::events::Event as QxEvent;
use rayon::prelude::*;
use crate::parse::{append_run_event, is_text_run, map_event, text_content};
use crate::scan::parse_doctype_entities;
pub struct ParallelXml {
buf: Buffer,
config: Config,
}
enum Buffer {
Owned(Cow<'static, [u8]>),
Mmap(memmap2::Mmap),
}
impl Buffer {
fn as_slice(&self) -> &[u8] {
match self {
Buffer::Owned(b) => b,
Buffer::Mmap(m) => m,
}
}
fn kind(&self) -> &'static str {
match self {
Buffer::Owned(_) => "owned",
Buffer::Mmap(_) => "mmap",
}
}
}
impl fmt::Debug for ParallelXml {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ParallelXml")
.field("buffer", &self.buf.kind())
.field("len", &self.buf.as_slice().len())
.field("config", &self.config)
.finish()
}
}
#[cfg(feature = "zstd")]
const ZSTD_MAGIC: [u8; 4] = [0x28, 0xB5, 0x2F, 0xFD];
#[cfg(feature = "zstd")]
fn is_zstd(bytes: &[u8]) -> bool {
bytes.starts_with(&ZSTD_MAGIC)
}
impl ParallelXml {
pub fn from_path(p: &Path) -> std::io::Result<Self> {
let file = std::fs::File::open(p)?;
let mmap = unsafe { memmap2::Mmap::map(&file)? };
#[cfg(feature = "zstd")]
if is_zstd(&mmap) {
let bytes = zstd::decode_all(&mmap[..])
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
return Ok(Self::from_owned(bytes));
}
Ok(Self {
buf: Buffer::Mmap(mmap),
config: Config::default(),
})
}
pub fn from_bytes(b: impl Into<Cow<'static, [u8]>>) -> Self {
Self {
buf: Buffer::Owned(b.into()),
config: Config::default(),
}
}
#[cfg(feature = "zstd")]
#[cfg_attr(docsrs, doc(cfg(feature = "zstd")))]
pub fn from_zstd_reader(reader: impl std::io::Read) -> Result<Self, XmlError> {
let bytes = zstd::decode_all(reader).map_err(XmlError::Io)?;
Ok(Self::from_owned(bytes))
}
#[cfg(feature = "zstd")]
#[cfg_attr(docsrs, doc(cfg(feature = "zstd")))]
pub fn from_zstd_bytes(compressed: &[u8]) -> Result<Self, XmlError> {
Self::from_zstd_reader(compressed)
}
#[cfg(feature = "zstd")]
fn from_owned(bytes: Vec<u8>) -> Self {
Self {
buf: Buffer::Owned(Cow::Owned(bytes)),
config: Config::default(),
}
}
pub fn with_config(mut self, cfg: Config) -> Self {
self.config = cfg;
self
}
pub fn index(&self) -> Result<ChunkIndex, XmlError> {
scan::scan_with(self.buf.as_slice(), &self.config.record_path)
}
pub fn par_for_each<F>(&self, f: F) -> Result<(), XmlError>
where
F: Fn(&Record) + Sync,
{
let buf = self.buf.as_slice();
let index = scan::scan_with(buf, &self.config.record_path)?;
let prelude = &index.prelude;
let make = |i: usize, r: &Range<usize>| Record {
bytes: &buf[r.clone()],
prelude: prelude.clone(),
index: i,
};
if self.run_sequential(buf.len(), index.records.len()) {
for (i, r) in index.records.iter().enumerate() {
f(&make(i, r));
}
} else {
index
.records
.par_iter()
.enumerate()
.for_each(|(i, r)| f(&make(i, r)));
}
Ok(())
}
pub fn map_collect<T, F>(&self, f: F) -> Result<Vec<T>, XmlError>
where
T: Send,
F: Fn(&Record) -> T + Sync,
{
let buf = self.buf.as_slice();
let index = scan::scan_with(buf, &self.config.record_path)?;
let prelude = &index.prelude;
let make = |i: usize, r: &Range<usize>| Record {
bytes: &buf[r.clone()],
prelude: prelude.clone(),
index: i,
};
let out = if self.run_sequential(buf.len(), index.records.len()) {
index
.records
.iter()
.enumerate()
.map(|(i, r)| f(&make(i, r)))
.collect()
} else {
index
.records
.par_iter()
.enumerate()
.map(|(i, r)| f(&make(i, r)))
.collect()
};
Ok(out)
}
pub fn try_par_for_each<F, E>(&self, f: F) -> Result<(), XmlError>
where
F: Fn(&Record) -> Result<(), E> + Sync,
E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
let buf = self.buf.as_slice();
let index = scan::scan_with(buf, &self.config.record_path)?;
let prelude = &index.prelude;
let one = |i: usize, r: &Range<usize>| {
let rec = Record {
bytes: &buf[r.clone()],
prelude: prelude.clone(),
index: i,
};
f(&rec).map_err(|e| XmlError::RecordError {
index: i,
source: e.into(),
})
};
if self.run_sequential(buf.len(), index.records.len()) {
index
.records
.iter()
.enumerate()
.try_for_each(|(i, r)| one(i, r))
} else {
index
.records
.par_iter()
.enumerate()
.try_for_each(|(i, r)| one(i, r))
}
}
pub fn try_map_collect<T, F, E>(&self, f: F) -> Result<Vec<T>, XmlError>
where
T: Send,
F: Fn(&Record) -> Result<T, E> + Sync,
E: Into<Box<dyn std::error::Error + Send + Sync>>,
{
let buf = self.buf.as_slice();
let index = scan::scan_with(buf, &self.config.record_path)?;
let prelude = &index.prelude;
let one = |i: usize, r: &Range<usize>| {
let rec = Record {
bytes: &buf[r.clone()],
prelude: prelude.clone(),
index: i,
};
f(&rec).map_err(|e| XmlError::RecordError {
index: i,
source: e.into(),
})
};
if self.run_sequential(buf.len(), index.records.len()) {
index
.records
.iter()
.enumerate()
.map(|(i, r)| one(i, r))
.collect()
} else {
index
.records
.par_iter()
.enumerate()
.map(|(i, r)| one(i, r))
.collect()
}
}
fn run_sequential(&self, byte_len: usize, record_count: usize) -> bool {
byte_len < self.config.parallel_threshold || record_count < self.config.min_records
}
pub fn sequential(&self) -> SeqReader<'_> {
SeqReader::new(self.buf.as_slice())
}
}
pub struct Record<'doc> {
bytes: &'doc [u8],
prelude: Arc<Prelude>,
index: usize,
}
impl<'doc> Record<'doc> {
pub(crate) fn new(bytes: &'doc [u8], prelude: Arc<Prelude>, index: usize) -> Self {
Self {
bytes,
prelude,
index,
}
}
pub fn events(&self) -> RecordReader<'doc> {
RecordReader::new(self.bytes, self.prelude.clone(), self.index)
}
pub fn index(&self) -> usize {
self.index
}
pub fn as_bytes(&self) -> &'doc [u8] {
self.bytes
}
}
impl fmt::Debug for Record<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Record")
.field("index", &self.index)
.field("len", &self.bytes.len())
.finish()
}
}
pub struct SeqReader<'doc> {
reader: Reader<&'doc [u8]>,
current: Option<QxEvent<'doc>>,
pending: Option<QxEvent<'doc>>,
prelude: Prelude,
}
impl fmt::Debug for SeqReader<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SeqReader")
.field("position", &self.reader.buffer_position())
.finish_non_exhaustive()
}
}
impl<'doc> SeqReader<'doc> {
fn new(bytes: &'doc [u8]) -> Self {
let mut reader = Reader::from_reader(bytes);
reader.config_mut().expand_empty_elements = true;
Self {
reader,
current: None,
pending: None,
prelude: Prelude {
encoding: Encoding::Utf8,
root_name: Box::default(),
namespaces: NamespaceContext::new(),
entities: HashMap::new(),
},
}
}
pub fn next_event(&mut self) -> Result<Option<Event<'_>>, XmlError> {
let Some(ev) = self.next_surfaced()? else {
return Ok(None);
};
if is_text_run(&ev) {
let next = self.read_raw()?;
let lone_literal =
matches!(ev, QxEvent::Text(_)) && !next.as_ref().is_some_and(is_text_run);
if lone_literal {
let QxEvent::Text(e) = ev else {
unreachable!("checked Text above")
};
let text = text_content(e);
self.pending = next;
return Ok(Some(Event::Text(text)));
}
let mut out = String::new();
append_run_event(&mut out, &ev, &self.prelude, 0)?;
let mut cur = next;
while let Some(ev) = cur {
if !is_text_run(&ev) {
self.pending = Some(ev);
break;
}
append_run_event(&mut out, &ev, &self.prelude, 0)?;
cur = self.read_raw()?;
}
return Ok(Some(Event::Text(Cow::Owned(out))));
}
self.current = Some(ev);
let event = map_event(
self.current.as_ref().expect("event stored above"),
&self.prelude,
0,
)?;
Ok(Some(event))
}
fn next_surfaced(&mut self) -> Result<Option<QxEvent<'doc>>, XmlError> {
loop {
match self.read_raw()? {
None => return Ok(None),
Some(QxEvent::DocType(e)) => {
parse_doctype_entities(e.as_bytes(), &mut self.prelude.entities);
}
Some(QxEvent::Comment(_) | QxEvent::PI(_) | QxEvent::Decl(_)) => continue,
Some(keep) => return Ok(Some(keep)),
}
}
}
fn read_raw(&mut self) -> Result<Option<QxEvent<'doc>>, XmlError> {
if let Some(ev) = self.pending.take() {
return Ok(Some(ev));
}
match self.reader.read_event() {
Ok(QxEvent::Eof) => Ok(None),
Ok(ev) => Ok(Some(ev)),
Err(_) => Err(XmlError::Malformed(self.reader.buffer_position() as usize)),
}
}
}
#[derive(Debug)]
pub enum XmlError {
Malformed(usize),
Encoding,
Io(std::io::Error),
UnsupportedDtd,
RecordError {
index: usize,
source: Box<dyn std::error::Error + Send + Sync>,
},
}
impl fmt::Display for XmlError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
XmlError::Malformed(pos) => write!(f, "malformed XML at byte {pos}"),
XmlError::Encoding => write!(f, "unsupported or unresolvable encoding"),
XmlError::Io(e) => write!(f, "I/O error: {e}"),
XmlError::UnsupportedDtd => {
write!(f, "external DTD / parameter entities are not supported")
}
XmlError::RecordError { index, source } => {
write!(f, "error in record {index}: {source}")
}
}
}
}
impl std::error::Error for XmlError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
XmlError::Io(e) => Some(e),
XmlError::RecordError { source, .. } => Some(&**source),
_ => None,
}
}
}
impl From<std::io::Error> for XmlError {
fn from(e: std::io::Error) -> Self {
XmlError::Io(e)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
fn build_doc(n: usize) -> String {
let mut s = String::from("<records>");
for i in 0..n {
s.push_str("<r>");
s.push_str(&i.to_string());
s.push_str("</r>");
}
s.push_str("</records>");
s
}
fn record_text(rec: &Record) -> String {
let mut reader = rec.events();
let mut out = String::new();
while let Some(ev) = reader.next_event().unwrap() {
if let Event::Text(t) = ev {
out.push_str(&t);
}
}
out
}
fn force_parallel() -> Config {
Config::new().with_parallel_threshold(0).with_min_records(0)
}
fn build_container_doc(n: usize) -> String {
let mut s = String::from("<root><manifest>meta</manifest><objects>");
for i in 0..n {
s.push_str("<object>");
s.push_str(&i.to_string());
s.push_str("</object>");
}
s.push_str("</objects></root>");
s
}
#[test]
fn record_path_frames_container_children_in_order() {
let n = 2000;
let px = ParallelXml::from_bytes(build_container_doc(n).into_bytes())
.with_config(force_parallel().with_record_path(["objects"]));
let got: Vec<usize> = px
.map_collect(|rec| record_text(rec).parse().unwrap())
.unwrap();
assert_eq!(got, (0..n).collect::<Vec<_>>());
}
#[test]
fn record_path_sequential_fallback_matches_parallel() {
let n = 100; let xml = build_container_doc(n);
let seq: Vec<usize> = ParallelXml::from_bytes(xml.clone().into_bytes())
.with_config(Config::new().with_record_path(["objects"]))
.map_collect(|rec| record_text(rec).parse().unwrap())
.unwrap();
let par: Vec<usize> = ParallelXml::from_bytes(xml.into_bytes())
.with_config(force_parallel().with_record_path(["objects"]))
.map_collect(|rec| record_text(rec).parse().unwrap())
.unwrap();
assert_eq!(seq, par);
assert_eq!(seq, (0..n).collect::<Vec<_>>());
}
#[test]
fn map_collect_preserves_document_order() {
let n = 2000;
let px = ParallelXml::from_bytes(build_doc(n).into_bytes()).with_config(force_parallel());
let got: Vec<usize> = px
.map_collect(|rec| record_text(rec).parse().unwrap())
.unwrap();
assert_eq!(got, (0..n).collect::<Vec<_>>());
}
#[test]
fn par_for_each_visits_every_record() {
let n = 1000;
let px = ParallelXml::from_bytes(build_doc(n).into_bytes()).with_config(force_parallel());
let sum = AtomicUsize::new(0);
let count = AtomicUsize::new(0);
px.par_for_each(|rec| {
sum.fetch_add(rec.index(), Ordering::Relaxed);
count.fetch_add(1, Ordering::Relaxed);
})
.unwrap();
assert_eq!(count.load(Ordering::Relaxed), n);
assert_eq!(sum.load(Ordering::Relaxed), n * (n - 1) / 2);
}
#[test]
fn small_input_fallback_matches_parallel() {
let n = 200;
let xml = build_doc(n);
let seq: Vec<usize> = ParallelXml::from_bytes(xml.clone().into_bytes())
.map_collect(|rec| record_text(rec).parse().unwrap())
.unwrap();
let par: Vec<usize> = ParallelXml::from_bytes(xml.into_bytes())
.with_config(force_parallel())
.map_collect(|rec| record_text(rec).parse().unwrap())
.unwrap();
assert_eq!(seq, par);
assert_eq!(seq, (0..n).collect::<Vec<_>>());
}
#[test]
fn map_collect_reports_scan_error() {
let px = ParallelXml::from_bytes(&b"<r><a></r>"[..]);
assert!(px.map_collect(|_| ()).is_err());
}
#[test]
fn try_map_collect_ok_preserves_order() {
let n = 300;
let px = ParallelXml::from_bytes(build_doc(n).into_bytes()).with_config(force_parallel());
let got: Vec<usize> = px
.try_map_collect(|rec| record_text(rec).parse::<usize>())
.unwrap();
assert_eq!(got, (0..n).collect::<Vec<_>>());
}
#[test]
fn try_map_collect_propagates_record_error() {
let xml = "<records><r>0</r><r>NaN</r><r>2</r></records>";
let px = ParallelXml::from_bytes(xml.as_bytes().to_vec()).with_config(force_parallel());
let res = px.try_map_collect(|rec| record_text(rec).parse::<usize>());
assert!(matches!(res, Err(XmlError::RecordError { index: 1, .. })));
}
#[test]
fn try_par_for_each_surfaces_record_error() {
let xml = "<records><r>0</r><r>oops</r></records>";
let px = ParallelXml::from_bytes(xml.as_bytes().to_vec());
let res = px.try_par_for_each(|rec| record_text(rec).parse::<usize>().map(|_| ()));
assert!(matches!(res, Err(XmlError::RecordError { index: 1, .. })));
}
#[test]
fn index_exposes_record_count() {
let px = ParallelXml::from_bytes(build_doc(5).into_bytes());
let idx = px.index().unwrap();
assert_eq!(idx.len(), 5);
}
#[test]
fn seq_reader_emits_all_events() {
let px = ParallelXml::from_bytes(&b"<r><a>x</a><b/></r>"[..]);
let mut sr = px.sequential();
let mut tags = Vec::new();
while let Some(ev) = sr.next_event().unwrap() {
tags.push(match ev {
Event::Start { name, .. } => {
format!("S:{}", name.as_ref())
}
Event::End { name } => {
format!("E:{}", name.as_ref())
}
Event::Text(t) => format!("T:{t}"),
Event::Cdata(c) => format!("C:{}", std::str::from_utf8(c).unwrap()),
});
}
assert_eq!(tags, ["S:r", "S:a", "T:x", "E:a", "S:b", "E:b", "E:r"]);
}
#[test]
fn seq_reader_resolves_doctype_entities() {
let doc = br#"<!DOCTYPE r [ <!ENTITY foo "BAR"> ]><r>&foo; & baz</r>"#;
let px = ParallelXml::from_bytes(&doc[..]);
let mut sr = px.sequential();
let mut text = String::new();
while let Some(ev) = sr.next_event().unwrap() {
if let Event::Text(t) = ev {
text.push_str(&t);
}
}
assert_eq!(text, "BAR & baz");
}
fn temp_path(tag: &str) -> std::path::PathBuf {
use std::sync::atomic::AtomicUsize;
static N: AtomicUsize = AtomicUsize::new(0);
let id = N.fetch_add(1, Ordering::Relaxed);
std::env::temp_dir().join(format!("pxml-{tag}-{}-{id}.bin", std::process::id()))
}
#[test]
fn from_path_reads_plain_xml() {
let path = temp_path("plain");
std::fs::write(&path, build_doc(40)).unwrap();
let doc = ParallelXml::from_path(&path).unwrap();
assert_eq!(doc.index().unwrap().len(), 40);
let _ = std::fs::remove_file(&path);
}
#[cfg(feature = "zstd")]
#[test]
fn zstd_bytes_roundtrip() {
let n = 500;
let xml = build_doc(n);
let compressed = zstd::encode_all(xml.as_bytes(), 3).unwrap();
assert!(
compressed.len() < xml.len(),
"input should actually compress"
);
let doc = ParallelXml::from_zstd_bytes(&compressed).unwrap();
let got: Vec<usize> = doc
.map_collect(|r| record_text(r).parse().unwrap())
.unwrap();
assert_eq!(got, (0..n).collect::<Vec<_>>());
}
#[cfg(feature = "zstd")]
#[test]
fn from_path_detects_and_decompresses_zstd() {
let n = 120;
let compressed = zstd::encode_all(build_doc(n).as_bytes(), 3).unwrap();
let path = temp_path("zstd");
std::fs::write(&path, &compressed).unwrap();
let doc = ParallelXml::from_path(&path).unwrap();
assert_eq!(doc.index().unwrap().len(), n);
let _ = std::fs::remove_file(&path);
}
}