use std::io::{Read, Seek, SeekFrom, Write};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use crate::compressors::IntegerCompressorBuilder;
use crate::decoders::ArithmeticDecoder;
use crate::decompressors::IntegerDecompressorBuilder;
use crate::encoders::ArithmeticEncoder;
pub use crate::errors::LasZipError;
use crate::las::nir::Nir;
use crate::las::point6::Point6;
use crate::las::rgb::RGB;
use crate::las::Point0;
use crate::record::{
LayeredPointRecordCompressor, LayeredPointRecordDecompressor, RecordCompressor,
RecordDecompressor, SequentialPointRecordCompressor, SequentialPointRecordDecompressor,
};
const DEFAULT_CHUNK_SIZE: usize = 50_000;
pub const LASZIP_USER_ID: &str = "laszip encoded";
pub const LASZIP_RECORD_ID: u16 = 22204;
pub const LASZIP_DESCRIPTION: &str = "http://laszip.org";
#[derive(Debug, Copy, Clone)]
struct Version {
major: u8,
minor: u8,
revision: u16,
}
impl Version {
fn read_from<R: Read>(src: &mut R) -> std::io::Result<Self> {
Ok(Self {
major: src.read_u8()?,
minor: src.read_u8()?,
revision: src.read_u16::<LittleEndian>()?,
})
}
fn write_to<W: Write>(&self, dst: &mut W) -> std::io::Result<()> {
dst.write_u8(self.major)?;
dst.write_u8(self.minor)?;
dst.write_u16::<LittleEndian>(self.revision)?;
Ok(())
}
}
#[derive(Debug, Copy, Clone)]
pub enum LazItemType {
Byte(u16),
Point10,
GpsTime,
RGB12,
Point14,
RGB14,
RGBNIR14,
Byte14(u16),
}
impl LazItemType {
fn size(&self) -> u16 {
match self {
LazItemType::Byte(size) => *size,
LazItemType::Point10 => Point0::SIZE as u16,
LazItemType::GpsTime => std::mem::size_of::<f64>() as u16,
LazItemType::RGB12 => RGB::SIZE as u16,
LazItemType::Point14 => Point6::SIZE as u16,
LazItemType::RGB14 => RGB::SIZE as u16,
LazItemType::RGBNIR14 => (RGB::SIZE + Nir::SIZE) as u16,
LazItemType::Byte14(size) => *size,
}
}
}
impl From<LazItemType> for u16 {
fn from(t: LazItemType) -> Self {
match t {
LazItemType::Byte(_) => 0,
LazItemType::Point10 => 6,
LazItemType::GpsTime => 7,
LazItemType::RGB12 => 8,
LazItemType::Point14 => 10,
LazItemType::RGB14 => 11,
LazItemType::RGBNIR14 => 12,
LazItemType::Byte14(_) => 14,
}
}
}
#[derive(Debug, Copy, Clone)]
pub struct LazItem {
pub(crate) item_type: LazItemType,
pub(crate) size: u16,
pub(crate) version: u16,
}
impl LazItem {
pub fn new(item_type: LazItemType, version: u16) -> Self {
let size = item_type.size();
Self {
item_type,
size,
version,
}
}
pub fn item_type(&self) -> LazItemType {
self.item_type
}
pub fn size(&self) -> u16 {
self.size
}
pub fn version(&self) -> u16 {
self.version
}
fn read_from<R: Read>(src: &mut R) -> crate::Result<Self> {
let item_type = src.read_u16::<LittleEndian>()?;
let size = src.read_u16::<LittleEndian>()?;
let item_type = match item_type {
0 => LazItemType::Byte(size),
6 => LazItemType::Point10,
7 => LazItemType::GpsTime,
8 => LazItemType::RGB12,
10 => LazItemType::Point14,
11 => LazItemType::RGB14,
12 => LazItemType::RGBNIR14,
14 => LazItemType::Byte14(size),
_ => return Err(LasZipError::UnknownLazItem(item_type)),
};
Ok(Self {
item_type,
size,
version: src.read_u16::<LittleEndian>()?,
})
}
fn write_to<W: Write>(&self, dst: &mut W) -> std::io::Result<()> {
dst.write_u16::<LittleEndian>(self.item_type.into())?;
dst.write_u16::<LittleEndian>(self.size)?;
dst.write_u16::<LittleEndian>(self.version)?;
Ok(())
}
}
pub struct LazItems(Vec<LazItem>);
macro_rules! define_trait_for_version {
($trait_name:ident, $trait_fn_name:ident) => {
pub trait $trait_name {
fn $trait_fn_name(num_extra_bytes: u16) -> Vec<LazItem>;
}
};
}
define_trait_for_version!(DefaultVersion, default_version);
define_trait_for_version!(Version1, version_1);
define_trait_for_version!(Version2, version_2);
define_trait_for_version!(Version3, version_3);
pub struct LazItemRecordBuilder {
items: Vec<LazItemType>,
}
impl LazItemRecordBuilder {
pub fn default_version_of<PointFormat: DefaultVersion>(num_extra_bytes: u16) -> Vec<LazItem> {
PointFormat::default_version(num_extra_bytes)
}
pub fn version_1_of<PointFormat: Version1>(num_extra_bytes: u16) -> Vec<LazItem> {
PointFormat::version_1(num_extra_bytes)
}
pub fn version_2_of<PointFormat: Version2>(num_extra_bytes: u16) -> Vec<LazItem> {
PointFormat::version_2(num_extra_bytes)
}
pub fn version_3_of<PointFormat: Version3>(num_extra_bytes: u16) -> Vec<LazItem> {
PointFormat::version_3(num_extra_bytes)
}
pub fn default_for_point_format_id(
point_format_id: u8,
num_extra_bytes: u16,
) -> crate::Result<Vec<LazItem>> {
use crate::las::{Point1, Point2, Point3, Point7, Point8};
match point_format_id {
0 => Ok(LazItemRecordBuilder::default_version_of::<Point0>(
num_extra_bytes,
)),
1 => Ok(LazItemRecordBuilder::default_version_of::<Point1>(
num_extra_bytes,
)),
2 => Ok(LazItemRecordBuilder::default_version_of::<Point2>(
num_extra_bytes,
)),
3 => Ok(LazItemRecordBuilder::default_version_of::<Point3>(
num_extra_bytes,
)),
6 => Ok(LazItemRecordBuilder::default_version_of::<Point6>(
num_extra_bytes,
)),
7 => Ok(LazItemRecordBuilder::default_version_of::<Point7>(
num_extra_bytes,
)),
8 => Ok(LazItemRecordBuilder::default_version_of::<Point8>(
num_extra_bytes,
)),
_ => Err(LasZipError::UnsupportedPointFormat(point_format_id)),
}
}
pub fn new() -> Self {
Self { items: vec![] }
}
pub fn add_item(&mut self, item_type: LazItemType) -> &mut Self {
self.items.push(item_type);
self
}
pub fn build(&self) -> Vec<LazItem> {
self.items
.iter()
.map(|item_type| {
let size = item_type.size();
let version = match item_type {
LazItemType::Byte(_) => 2,
LazItemType::Point10 => 2,
LazItemType::GpsTime => 2,
LazItemType::RGB12 => 2,
LazItemType::Point14 => 3,
LazItemType::RGB14 => 3,
LazItemType::RGBNIR14 => 3,
LazItemType::Byte14(_) => 3,
};
LazItem {
item_type: *item_type,
size,
version,
}
})
.collect()
}
}
fn read_laz_items_from<R: Read>(mut src: &mut R) -> crate::Result<Vec<LazItem>> {
let num_items = src.read_u16::<LittleEndian>()?;
let mut items = Vec::<LazItem>::with_capacity(num_items as usize);
for _ in 0..num_items {
items.push(LazItem::read_from(&mut src)?)
}
Ok(items)
}
fn write_laz_items_to<W: Write>(laz_items: &Vec<LazItem>, mut dst: &mut W) -> std::io::Result<()> {
dst.write_u16::<LittleEndian>(laz_items.len() as u16)?;
for item in laz_items {
item.write_to(&mut dst)?;
}
Ok(())
}
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum CompressorType {
None = 0,
PointWise = 1,
PointWiseChunked = 2,
LayeredChunked = 3,
}
impl CompressorType {
fn from_u16(t: u16) -> Option<Self> {
match t {
0 => Some(CompressorType::None),
1 => Some(CompressorType::PointWise),
2 => Some(CompressorType::PointWiseChunked),
3 => Some(CompressorType::LayeredChunked),
_ => None,
}
}
}
impl Default for CompressorType {
fn default() -> Self {
CompressorType::PointWiseChunked
}
}
#[derive(Debug, Clone)]
pub struct LazVlr {
compressor: CompressorType,
coder: u16,
version: Version,
options: u32,
chunk_size: u32,
number_of_special_evlrs: i64,
offset_to_special_evlrs: i64,
items: Vec<LazItem>,
}
impl LazVlr {
pub fn from_laz_items(items: Vec<LazItem>) -> Self {
let first_item = items
.first()
.expect("Vec<LazItem> should at least have one element");
let compressor = match first_item.version {
1 | 2 => CompressorType::PointWiseChunked,
3 | 4 => CompressorType::LayeredChunked,
_ => panic!("Unknown laz_item version"),
};
Self {
compressor,
coder: 0,
version: Version {
major: 2,
minor: 2,
revision: 0,
},
options: 0,
chunk_size: DEFAULT_CHUNK_SIZE as u32,
number_of_special_evlrs: -1,
offset_to_special_evlrs: -1,
items,
}
}
pub fn from_buffer(record_data: &[u8]) -> crate::Result<Self> {
let mut cursor = std::io::Cursor::new(record_data);
Self::read_from(&mut cursor)
}
pub fn read_from<R: Read>(mut src: &mut R) -> crate::Result<Self> {
let compressor_type = src.read_u16::<LittleEndian>()?;
let compressor = match CompressorType::from_u16(compressor_type) {
Some(c) => c,
None => return Err(LasZipError::UnknownCompressorType(compressor_type)),
};
Ok(Self {
compressor,
coder: src.read_u16::<LittleEndian>()?,
version: Version::read_from(&mut src)?,
options: src.read_u32::<LittleEndian>()?,
chunk_size: src.read_u32::<LittleEndian>()?,
number_of_special_evlrs: src.read_i64::<LittleEndian>()?,
offset_to_special_evlrs: src.read_i64::<LittleEndian>()?,
items: read_laz_items_from(&mut src)?,
})
}
pub fn write_to<W: Write>(&self, mut dst: &mut W) -> std::io::Result<()> {
dst.write_u16::<LittleEndian>(self.compressor as u16)?;
dst.write_u16::<LittleEndian>(self.coder)?;
self.version.write_to(&mut dst)?;
dst.write_u32::<LittleEndian>(self.options)?;
dst.write_u32::<LittleEndian>(self.chunk_size)?;
dst.write_i64::<LittleEndian>(self.number_of_special_evlrs)?;
dst.write_i64::<LittleEndian>(self.offset_to_special_evlrs)?;
write_laz_items_to(&self.items, &mut dst)?;
Ok(())
}
pub fn chunk_size(&self) -> u32 {
self.chunk_size
}
pub fn items(&self) -> &Vec<LazItem> {
&self.items
}
pub fn items_size(&self) -> u64 {
u64::from(self.items.iter().map(|item| item.size).sum::<u16>())
}
}
impl Default for LazVlr {
fn default() -> Self {
Self {
compressor: Default::default(),
coder: 0,
version: Version {
major: 2,
minor: 2,
revision: 0,
},
options: 0,
chunk_size: DEFAULT_CHUNK_SIZE as u32,
number_of_special_evlrs: -1,
offset_to_special_evlrs: -1,
items: vec![],
}
}
}
pub struct LazVlrBuilder {
laz_vlr: LazVlr,
}
impl LazVlrBuilder {
pub fn new() -> Self {
Self {
laz_vlr: Default::default(),
}
}
pub fn with_chunk_size(mut self, chunk_size: u32) -> Self {
self.laz_vlr.chunk_size = chunk_size;
self
}
pub fn with_laz_items(mut self, laz_items: Vec<LazItem>) -> Self {
self.laz_vlr.items = laz_items;
self
}
pub fn build(self) -> LazVlr {
self.laz_vlr
}
}
fn record_decompressor_from_laz_items<'a, R: Read + Seek + Send + 'a>(
items: &Vec<LazItem>,
input: R,
) -> crate::Result<Box<dyn RecordDecompressor<R> + Send + 'a>> {
let first_item = items
.get(0)
.expect("There should be at least one LazItem to be able to create a RecordDecompressor");
let mut decompressor = match first_item.version {
1 | 2 => {
let decompressor = SequentialPointRecordDecompressor::new(input);
Box::new(decompressor) as Box<dyn RecordDecompressor<R> + Send>
}
3 | 4 => {
let decompressor = LayeredPointRecordDecompressor::new(input);
Box::new(decompressor) as Box<dyn RecordDecompressor<R> + Send>
}
_ => {
return Err(LasZipError::UnsupportedLazItemVersion(
first_item.item_type,
first_item.version,
));
}
};
decompressor.set_fields_from(items)?;
Ok(decompressor)
}
fn record_compressor_from_laz_items<'a, W: Write + Send + 'a>(
items: &Vec<LazItem>,
output: W,
) -> crate::Result<Box<dyn RecordCompressor<W> + Send + 'a>> {
let first_item = items
.get(0)
.expect("There should be at least one LazItem to be able to create a RecordCompressor");
let mut compressor = match first_item.version {
1 | 2 => {
let compressor = SequentialPointRecordCompressor::new(output);
Box::new(compressor) as Box<dyn RecordCompressor<W> + Send>
}
3 | 4 => {
let compressor = LayeredPointRecordCompressor::new(output);
Box::new(compressor) as Box<dyn RecordCompressor<W> + Send>
}
_ => {
return Err(LasZipError::UnsupportedLazItemVersion(
first_item.item_type,
first_item.version,
));
}
};
compressor.set_fields_from(items)?;
Ok(compressor)
}
pub fn read_chunk_table<R: Read + Seek>(src: &mut R) -> Option<std::io::Result<Vec<u64>>> {
let current_pos = match src.seek(SeekFrom::Current(0)) {
Ok(p) => p,
Err(e) => return Some(Err(e)),
};
let offset_to_chunk_table = match src.read_i64::<LittleEndian>() {
Ok(p) => p,
Err(e) => return Some(Err(e)),
};
if offset_to_chunk_table >= 0 && offset_to_chunk_table as u64 <= current_pos {
None
} else {
Some(read_chunk_table_at_offset(src, offset_to_chunk_table))
}
}
fn read_chunk_table_at_offset<R: Read + Seek>(
mut src: &mut R,
mut offset_to_chunk_table: i64,
) -> std::io::Result<Vec<u64>> {
let current_pos = src.seek(SeekFrom::Current(0))?;
if offset_to_chunk_table == -1 {
src.seek(SeekFrom::End(-8))?;
offset_to_chunk_table = src.read_i64::<LittleEndian>()?;
}
src.seek(SeekFrom::Start(offset_to_chunk_table as u64))?;
let _version = src.read_u32::<LittleEndian>()?;
let number_of_chunks = src.read_u32::<LittleEndian>()?;
let mut chunk_sizes = vec![0u64; number_of_chunks as usize];
let mut decompressor = IntegerDecompressorBuilder::new()
.bits(32)
.contexts(2)
.build_initialized();
let mut decoder = ArithmeticDecoder::new(&mut src);
decoder.read_init_bytes()?;
for i in 1..=number_of_chunks {
chunk_sizes[(i - 1) as usize] = decompressor.decompress(
&mut decoder,
if i > 1 {
chunk_sizes[(i - 2) as usize]
} else {
0
} as i32,
1,
)? as u64;
}
src.seek(SeekFrom::Start(current_pos))?;
Ok(chunk_sizes)
}
pub struct LasZipDecompressor<'a, R: Read + Seek + 'a> {
vlr: LazVlr,
record_decompressor: Box<dyn RecordDecompressor<R> + Send + 'a>,
chunk_points_read: u32,
offset_to_chunk_table: i64,
data_start: u64,
chunk_table: Option<Vec<u64>>,
}
impl<'a, R: Read + Seek + Send + 'a> LasZipDecompressor<'a, R> {
pub fn new(mut source: R, vlr: LazVlr) -> crate::Result<Self> {
if vlr.compressor != CompressorType::PointWiseChunked
&& vlr.compressor != CompressorType::LayeredChunked
{
return Err(LasZipError::UnsupportedCompressorType(vlr.compressor));
}
let offset_to_chunk_table = source.read_i64::<LittleEndian>()?;
let data_start = source.seek(SeekFrom::Current(0))?;
let record_decompressor = record_decompressor_from_laz_items(&vlr.items, source)?;
Ok(Self {
vlr,
record_decompressor,
chunk_points_read: 0,
offset_to_chunk_table,
data_start,
chunk_table: None,
})
}
pub fn new_with_record_data(source: R, laszip_vlr_record_data: &[u8]) -> crate::Result<Self> {
let vlr = LazVlr::from_buffer(laszip_vlr_record_data)?;
Self::new(source, vlr)
}
pub fn decompress_one(&mut self, mut out: &mut [u8]) -> std::io::Result<()> {
if self.chunk_points_read == self.vlr.chunk_size {
self.reset_for_new_chunk();
}
self.record_decompressor.decompress_next(&mut out)?;
self.chunk_points_read += 1;
Ok(())
}
pub fn decompress_many(&mut self, out: &mut [u8]) -> std::io::Result<()> {
for point in out.chunks_exact_mut(self.vlr.items_size() as usize) {
self.decompress_one(point)?;
}
Ok(())
}
pub fn vlr(&self) -> &LazVlr {
&self.vlr
}
pub fn seek(&mut self, point_idx: u64) -> std::io::Result<()> {
if let Some(chunk_table) = &self.chunk_table {
let chunk_of_point = point_idx / self.vlr.chunk_size as u64;
let delta = point_idx % self.vlr.chunk_size as u64;
if chunk_of_point == (chunk_table.len() - 1) as u64 {
if self.offset_to_chunk_table == -1 {
unreachable!("unexpected offset to chunk table");
}
let mut tmp_out = vec![0u8; self.record_decompressor.record_size()];
self.record_decompressor
.get_mut()
.seek(SeekFrom::Start(chunk_table[chunk_of_point as usize] as u64))?;
self.reset_for_new_chunk();
for _i in 0..delta {
self.decompress_one(&mut tmp_out)?;
let current_pos = self
.record_decompressor
.get_mut()
.seek(SeekFrom::Current(0))?;
if current_pos >= self.offset_to_chunk_table as u64 {
self.record_decompressor.get_mut().seek(SeekFrom::End(0))?;
return Ok(());
}
}
} else if let Some(start_of_chunk) = chunk_table.get(chunk_of_point as usize) {
self.record_decompressor
.get_mut()
.seek(SeekFrom::Start(*start_of_chunk as u64))?;
self.reset_for_new_chunk();
let mut tmp_out = vec![0u8; self.record_decompressor.record_size()];
for _i in 0..delta {
self.decompress_one(&mut tmp_out)?;
}
} else {
self.record_decompressor.get_mut().seek(SeekFrom::End(0))?;
}
Ok(())
} else {
self.read_chunk_table()?;
self.seek(point_idx)
}
}
fn reset_for_new_chunk(&mut self) {
self.chunk_points_read = 0;
self.record_decompressor.reset();
self.record_decompressor
.set_fields_from(&self.vlr.items)
.unwrap();
}
fn read_chunk_table(&mut self) -> std::io::Result<()> {
let stream = self.record_decompressor.get_mut();
let chunk_sizes = read_chunk_table_at_offset(stream, self.offset_to_chunk_table)?;
let number_of_chunks = chunk_sizes.len();
let mut chunk_starts = vec![0u64; number_of_chunks as usize];
chunk_starts[0] = self.data_start;
for i in 1..number_of_chunks {
chunk_starts[i as usize] =
chunk_sizes[(i - 1) as usize] + chunk_starts[(i - 1) as usize];
}
self.chunk_table = Some(chunk_starts);
Ok(())
}
pub fn into_inner(self) -> R {
self.record_decompressor.box_into_inner()
}
pub fn get_mut(&mut self) -> &mut R {
self.record_decompressor.get_mut()
}
pub fn get(&self) -> &R {
self.record_decompressor.get()
}
}
pub fn write_chunk_table<W: Write>(
mut stream: &mut W,
chunk_table: &Vec<usize>,
) -> std::io::Result<()> {
stream.write_u32::<LittleEndian>(0)?;
stream.write_u32::<LittleEndian>(chunk_table.len() as u32)?;
let mut encoder = ArithmeticEncoder::new(&mut stream);
let mut compressor = IntegerCompressorBuilder::new()
.bits(32)
.contexts(2)
.build_initialized();
let mut predictor = 0;
for chunk_size in chunk_table {
compressor.compress(&mut encoder, predictor, (*chunk_size) as i32, 1)?;
predictor = (*chunk_size) as i32;
}
encoder.done()?;
Ok(())
}
fn update_chunk_table_offset<W: Write + Seek>(
dst: &mut W,
offset_pos: SeekFrom,
) -> std::io::Result<()> {
let start_of_chunk_table_pos = dst.seek(SeekFrom::Current(0))?;
dst.seek(offset_pos)?;
dst.write_i64::<LittleEndian>(start_of_chunk_table_pos as i64)?;
dst.seek(SeekFrom::Start(start_of_chunk_table_pos))?;
Ok(())
}
pub struct LasZipCompressor<'a, W: Write + Send + 'a> {
vlr: LazVlr,
record_compressor: Box<dyn RecordCompressor<W> + Send + 'a>,
first_point: bool,
chunk_point_written: u32,
chunk_sizes: Vec<usize>,
last_chunk_pos: u64,
start_pos: u64,
}
impl<'a, W: Write + Seek + Send + 'a> LasZipCompressor<'a, W> {
pub fn new(output: W, vlr: LazVlr) -> crate::Result<Self> {
let record_compressor = record_compressor_from_laz_items(&vlr.items, output)?;
Ok(Self {
vlr,
record_compressor,
first_point: true,
chunk_point_written: 0,
chunk_sizes: vec![],
last_chunk_pos: 0,
start_pos: 0,
})
}
pub fn from_laz_items(output: W, items: Vec<LazItem>) -> crate::Result<Self> {
let vlr = LazVlr::from_laz_items(items);
Self::new(output, vlr)
}
pub fn compress_one(&mut self, input: &[u8]) -> std::io::Result<()> {
if self.first_point {
let stream = self.record_compressor.get_mut();
self.start_pos = stream.seek(SeekFrom::Current(0))?;
stream.write_i64::<LittleEndian>(-1)?;
self.last_chunk_pos = self.start_pos + std::mem::size_of::<i64>() as u64;
self.first_point = false;
}
if self.chunk_point_written == self.vlr.chunk_size {
self.record_compressor.done()?;
self.record_compressor.reset();
self.record_compressor
.set_fields_from(&self.vlr.items)
.unwrap();
self.update_chunk_table()?;
self.chunk_point_written = 0;
}
self.record_compressor.compress_next(&input)?;
self.chunk_point_written += 1;
Ok(())
}
pub fn compress_many(&mut self, input: &[u8]) -> std::io::Result<()> {
for point in input.chunks_exact(self.vlr.items_size() as usize) {
self.compress_one(point)?;
}
Ok(())
}
pub fn done(&mut self) -> std::io::Result<()> {
self.record_compressor.done()?;
self.update_chunk_table()?;
let stream = self.record_compressor.get_mut();
update_chunk_table_offset(stream, SeekFrom::Start(self.start_pos))?;
write_chunk_table(stream, &self.chunk_sizes)?;
Ok(())
}
pub fn vlr(&self) -> &LazVlr {
&self.vlr
}
pub fn into_inner(self) -> W {
self.record_compressor.box_into_inner()
}
pub fn get_mut(&mut self) -> &mut W {
self.record_compressor.get_mut()
}
pub fn get(&self) -> &W {
self.record_compressor.get()
}
fn update_chunk_table(&mut self) -> std::io::Result<()> {
let current_pos = self
.record_compressor
.get_mut()
.seek(SeekFrom::Current(0))?;
self.chunk_sizes
.push((current_pos - self.last_chunk_pos) as usize);
self.last_chunk_pos = current_pos;
Ok(())
}
}
pub fn compress_buffer<W: Write + Seek + Send>(
dst: &mut W,
uncompressed_points: &[u8],
laz_vlr: LazVlr,
) -> crate::Result<()> {
let mut compressor = LasZipCompressor::new(dst, laz_vlr)?;
let point_size = compressor.vlr().items_size() as usize;
if uncompressed_points.len() % point_size != 0 {
Err(LasZipError::BufferLenNotMultipleOfPointSize {
buffer_len: uncompressed_points.len(),
point_size,
})
} else {
compressor.compress_many(uncompressed_points)?;
compressor.done()?;
Ok(())
}
}
pub fn decompress_buffer(
compressed_points_data: &[u8],
decompressed_points: &mut [u8],
laz_vlr: LazVlr,
) -> crate::Result<()> {
let point_size = laz_vlr.items_size() as usize;
if decompressed_points.len() % point_size != 0 {
Err(LasZipError::BufferLenNotMultipleOfPointSize {
buffer_len: decompressed_points.len(),
point_size,
})
} else {
let src = std::io::Cursor::new(compressed_points_data);
LasZipDecompressor::new(src, laz_vlr).and_then(|mut decompressor| {
decompressor.decompress_many(decompressed_points)?;
Ok(())
})
}
}
#[cfg(feature = "parallel")]
pub fn par_compress_buffer<W: Write + Seek>(
dst: &mut W,
uncompressed_points: &[u8],
laz_vlr: &LazVlr,
) -> crate::Result<()> {
let start_pos = dst.seek(SeekFrom::Current(0))?;
dst.write_i64::<LittleEndian>(start_pos as i64)?;
let chunk_sizes = par_compress(dst, uncompressed_points, laz_vlr)?;
update_chunk_table_offset(dst, SeekFrom::Start(start_pos))?;
write_chunk_table(dst, &chunk_sizes)?;
Ok(())
}
#[cfg(feature = "parallel")]
pub fn par_compress<W: Write>(
dst: &mut W,
uncompressed_points: &[u8],
laz_vlr: &LazVlr,
) -> crate::Result<Vec<usize>> {
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use std::io::Cursor;
let point_size = laz_vlr.items_size() as usize;
if uncompressed_points.len() % point_size != 0 {
Err(LasZipError::BufferLenNotMultipleOfPointSize {
buffer_len: uncompressed_points.len(),
point_size,
})
} else {
let points_per_chunk = laz_vlr.chunk_size as usize;
let chunk_size_in_bytes = points_per_chunk * point_size;
let all_slices = uncompressed_points
.chunks(chunk_size_in_bytes)
.collect::<Vec<_>>();
let chunks = all_slices
.into_par_iter()
.map(|slc| {
let mut record_compressor = record_compressor_from_laz_items(
&laz_vlr.items,
Cursor::new(Vec::<u8>::new()),
)?;
for raw_point in slc.chunks_exact(point_size) {
record_compressor.compress_next(raw_point)?;
}
record_compressor.done()?;
Ok(record_compressor.box_into_inner())
})
.collect::<Vec<crate::Result<Cursor<Vec<u8>>>>>();
let mut chunk_sizes = Vec::<usize>::with_capacity(chunks.len());
for chunk_result in chunks {
let chunk = chunk_result?;
chunk_sizes.push(chunk.get_ref().len());
dst.write_all(chunk.get_ref())?;
}
Ok(chunk_sizes)
}
}
#[cfg(feature = "parallel")]
pub fn par_decompress_buffer(
compressed_points_data: &[u8],
decompressed_points: &mut [u8],
laz_vlr: &LazVlr,
) -> crate::Result<()> {
let point_size = laz_vlr.items_size() as usize;
if decompressed_points.len() % point_size != 0 {
Err(LasZipError::BufferLenNotMultipleOfPointSize {
buffer_len: decompressed_points.len(),
point_size,
})
} else {
let mut cursor = std::io::Cursor::new(compressed_points_data);
let offset_to_chunk_table = cursor.read_i64::<LittleEndian>()?;
let chunk_sizes = read_chunk_table_at_offset(&mut cursor, offset_to_chunk_table)?;
let compressed_points =
&compressed_points_data[std::mem::size_of::<i64>()..offset_to_chunk_table as usize];
par_decompress(
compressed_points,
decompressed_points,
laz_vlr,
&chunk_sizes,
)
}
}
#[cfg(feature = "parallel")]
fn par_decompress(
compressed_points: &[u8],
decompressed_points: &mut [u8],
laz_vlr: &LazVlr,
chunk_sizes: &[u64],
) -> crate::Result<()> {
use crate::byteslice::ChunksIrregular;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
let point_size = laz_vlr.items_size() as usize;
let decompressed_chunk_size = laz_vlr.chunk_size as usize * point_size;
let sizes = chunk_sizes
.iter()
.map(|s| *s as usize)
.collect::<Vec<usize>>();
let input_chunks_iter = ChunksIrregular::new(compressed_points, &sizes);
let output_chunks_iter = decompressed_points.chunks_mut(decompressed_chunk_size as usize);
let decompression_jobs: Vec<(&[u8], &mut [u8])> =
input_chunks_iter.zip(output_chunks_iter).collect();
decompression_jobs
.into_par_iter()
.map(|(chunk_in, chunk_out)| {
let src = std::io::Cursor::new(chunk_in);
let mut record_decompressor = record_decompressor_from_laz_items(laz_vlr.items(), src)?;
for raw_point in chunk_out.chunks_exact_mut(point_size) {
record_decompressor.decompress_next(raw_point)?;
}
Ok(())
})
.collect::<crate::Result<()>>()?;
Ok(())
}
#[cfg(feature = "parallel")]
pub fn par_decompress_all_from_file_greedy(
src: &mut std::io::BufReader<std::fs::File>,
points_out: &mut [u8],
laz_vlr: &LazVlr,
) -> crate::Result<()> {
let point_size = laz_vlr.items_size() as usize;
if points_out.len() % point_size != 0 {
Err(LasZipError::BufferLenNotMultipleOfPointSize {
buffer_len: points_out.len(),
point_size,
})
} else {
let chunk_table = read_chunk_table(src).ok_or(LasZipError::MissingChunkTable)??;
let point_data_size = chunk_table.iter().copied().sum::<u64>();
let mut compressed_points = vec![0u8; point_data_size as usize];
src.read_exact(&mut compressed_points)?;
par_decompress(&compressed_points, points_out, laz_vlr, &chunk_table)
}
}
#[cfg(feature = "parallel")]
pub struct ParLasZipCompressor<W> {
vlr: LazVlr,
chunk_table: Vec<usize>,
table_offset: u64,
rest: Vec<u8>,
internal_buffer: Vec<u8>,
dest: W,
}
#[cfg(feature = "parallel")]
impl<W: Write + Seek> ParLasZipCompressor<W> {
pub fn new(dest: W, vlr: LazVlr) -> crate::Result<Self> {
let mut myself = Self {
vlr,
chunk_table: vec![],
table_offset: 0,
rest: vec![],
internal_buffer: vec![],
dest,
};
myself.reserve_chunk_table_offset()?;
Ok(myself)
}
fn reserve_chunk_table_offset(&mut self) -> std::io::Result<()> {
self.table_offset = self.dest.seek(SeekFrom::Current(0))?;
self.dest
.write_i64::<LittleEndian>(self.table_offset as i64)
}
pub fn compress_many(&mut self, points: &[u8]) -> std::io::Result<()> {
let point_size = self.vlr.items_size() as usize;
debug_assert_eq!(self.rest.len() % point_size, 0);
let chunk_size_in_bytes = self.vlr.chunk_size() as usize * point_size;
let num_chunk = (self.rest.len() + points.len()) / chunk_size_in_bytes;
let num_bytes_not_fitting = (self.rest.len() + points.len()) % chunk_size_in_bytes;
self.internal_buffer
.resize(num_chunk * chunk_size_in_bytes, 0u8);
if num_chunk > 0 {
self.internal_buffer[..self.rest.len()].copy_from_slice(&self.rest);
self.internal_buffer[self.rest.len()..]
.copy_from_slice(&points[..points.len() - num_bytes_not_fitting]);
self.rest.resize(num_bytes_not_fitting, 0u8);
self.rest
.copy_from_slice(&points[points.len() - num_bytes_not_fitting..]);
let chunk_sizes =
par_compress(&mut self.dest, &self.internal_buffer, &self.vlr).unwrap();
chunk_sizes
.iter()
.copied()
.map(|size| size as usize)
.for_each(|size| self.chunk_table.push(size));
} else {
for b in points {
self.rest.push(*b);
}
}
Ok(())
}
pub fn done(&mut self) -> crate::Result<()> {
let chunk_sizes = par_compress(&mut self.dest, &self.rest, &self.vlr)?;
chunk_sizes
.iter()
.copied()
.map(|size| size as usize)
.for_each(|size| self.chunk_table.push(size));
update_chunk_table_offset(&mut self.dest, SeekFrom::Start(self.table_offset))?;
write_chunk_table(&mut self.dest, &self.chunk_table)?;
Ok(())
}
pub fn vlr(&self) -> &LazVlr {
&self.vlr
}
pub fn into_inner(self) -> W {
self.dest
}
pub fn get_mut(&mut self) -> &mut W {
&mut self.dest
}
pub fn get(&self) -> &W {
&self.dest
}
}
#[cfg(feature = "parallel")]
pub struct ParLasZipDecompressor<R> {
vlr: LazVlr,
chunk_table: Vec<u64>,
last_chunk_read: usize,
rest: std::io::Cursor<Vec<u8>>,
internal_buffer: Vec<u8>,
outernal_buffer: Vec<u8>,
source: R,
}
#[cfg(feature = "parallel")]
impl<R: Read + Seek> ParLasZipDecompressor<R> {
pub fn new(mut source: R, vlr: LazVlr) -> crate::Result<Self> {
let chunk_table = read_chunk_table(&mut source).ok_or(LasZipError::MissingChunkTable)??;
Ok(Self {
source,
vlr,
chunk_table,
rest: std::io::Cursor::<Vec<u8>>::new(vec![]),
internal_buffer: vec![],
outernal_buffer: vec![],
last_chunk_read: 0,
})
}
pub fn decompress_many(&mut self, out: &mut [u8]) -> crate::Result<()> {
let point_size = self.vlr.items_size() as usize;
assert_eq!(out.len() % point_size, 0);
let num_bytes_in_rest = self.rest.get_ref().len() - self.rest.position() as usize;
debug_assert!(num_bytes_in_rest % point_size == 0);
if num_bytes_in_rest >= out.len() {
self.rest.read(out)?;
} else {
let num_bytes_in_chunk =
self.vlr.chunk_size() as usize * self.vlr.items_size() as usize;
let num_chunks_to_decompress = ((out.len() - num_bytes_in_rest) as f32
/ num_bytes_in_chunk as f32)
.ceil() as usize;
let chunk_sizes = &self.chunk_table
[self.last_chunk_read..self.last_chunk_read + num_chunks_to_decompress];
let bytes_to_read = chunk_sizes.iter().copied().sum::<u64>() as usize;
self.internal_buffer.resize(bytes_to_read, 0u8);
self.outernal_buffer
.resize(num_chunks_to_decompress * num_bytes_in_chunk, 0u8);
self.source.read(&mut self.internal_buffer)?;
if self.last_chunk_read + num_chunks_to_decompress < self.chunk_table.len() {
par_decompress(
&self.internal_buffer,
&mut self.outernal_buffer,
&self.vlr,
&chunk_sizes,
)?;
} else {
par_decompress(
&self.internal_buffer,
&mut self.outernal_buffer,
&self.vlr,
&chunk_sizes[..chunk_sizes.len() - 1],
)?;
let last_chunk_start = bytes_to_read - (*chunk_sizes.last().unwrap() as usize);
let last_chunk_source =
std::io::Cursor::new(&self.internal_buffer[last_chunk_start..]);
let mut decompressor =
record_decompressor_from_laz_items(self.vlr.items(), last_chunk_source)?;
debug_assert_eq!(
self.outernal_buffer[(num_chunks_to_decompress - 1) * num_bytes_in_chunk..]
.len()
% point_size,
0
);
let mut num_decompressed_bytes_in_last_chunk = 0;
for point in self.outernal_buffer
[(num_chunks_to_decompress - 1) * num_bytes_in_chunk..]
.chunks_exact_mut(point_size)
{
if let Err(error) = decompressor.decompress_next(point) {
if error.kind() == std::io::ErrorKind::UnexpectedEof {
break;
} else {
return Err(error.into());
}
} else {
num_decompressed_bytes_in_last_chunk += point_size;
}
}
self.outernal_buffer.resize(
(num_chunks_to_decompress - 1) * num_bytes_in_chunk
+ num_decompressed_bytes_in_last_chunk,
0u8,
);
}
let num_bytes_not_fitting =
(self.outernal_buffer.len() + num_bytes_in_rest) - out.len();
self.rest.read(&mut out[..num_bytes_in_rest])?;
out[num_bytes_in_rest..].copy_from_slice(
&self.outernal_buffer[..self.outernal_buffer.len() - num_bytes_not_fitting],
);
debug_assert_eq!(
self.rest.position() as usize,
self.rest.get_ref().len(),
"The rest was not consumed"
);
{
let rest_vec = self.rest.get_mut();
rest_vec.resize(num_bytes_not_fitting, 0u8);
rest_vec.copy_from_slice(
&self.outernal_buffer[self.outernal_buffer.len() - num_bytes_not_fitting..],
);
self.rest.set_position(0);
}
self.last_chunk_read += num_chunks_to_decompress;
}
Ok(())
}
pub fn into_inner(self) -> R {
self.source
}
pub fn get_mut(&mut self) -> &mut R {
&mut self.source
}
pub fn get(&self) -> &R {
&self.source
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_create_laz_items() {
assert_eq!(
LazItemRecordBuilder::new()
.add_item(LazItemType::Point10)
.build()
.len(),
1
);
}
}