1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
use crate::laszip::details::record_decompressor_from_laz_items;
use crate::laszip::{ChunkTable, CompressorType};
use crate::{LasZipCompressor, LasZipError, LazCompressor, LazVlr};
use std::io::{Cursor, Read, Seek, SeekFrom, Write};
pub(crate) fn prepare_compressor_for_appending<W, F, F2, Compressor>(
mut data: W,
vlr: LazVlr,
compressor_creator: F,
get_mut_dest_of_compressor: F2,
) -> crate::Result<(Compressor, ChunkTable)>
where
W: Write + Read + Seek,
F: FnOnce(W, LazVlr) -> crate::Result<Compressor>,
F2: FnOnce(&mut Compressor) -> &mut W,
Compressor: LazCompressor,
{
// Technically we could support PointWise compressor
// But it's old and rare so not much point to do so
if vlr.compressor != CompressorType::PointWiseChunked
&& vlr.compressor != CompressorType::LayeredChunked
{
return Err(LasZipError::UnsupportedCompressorType(vlr.compressor));
}
let start_of_data = data.seek(SeekFrom::Current(0))?;
let mut chunk_table = ChunkTable::read_from(&mut data, &vlr)?;
let mut data_to_recompress = vec![];
if !vlr.uses_variable_size_chunks() && !chunk_table.is_empty() {
// In PointWiseChunked, we don't know if the last chunk is complete or not
// so we read it, rewrite it so the compressor is in the right state to append points
let size_of_all_other_chunks = chunk_table.chunk_position(chunk_table.len() - 1).unwrap(); // We know the chunk table is not empty
let size_of_last_chunk = chunk_table[chunk_table.len() - 1].byte_count;
let mut last_chunk_data = vec![0u8; size_of_last_chunk as usize];
let last_chunk_pos = size_of_all_other_chunks.try_into().unwrap();
data.seek(SeekFrom::Current(last_chunk_pos))?;
data.read_exact(&mut last_chunk_data)?;
let mut last_chunk_data = Cursor::new(last_chunk_data);
let mut decompressor =
record_decompressor_from_laz_items(vlr.items(), &mut last_chunk_data)?;
data_to_recompress.resize(
(chunk_table[chunk_table.len() - 1].point_count * vlr.items_size()) as usize,
0,
);
// We cannot trust the point count of the chunk entry
// so we use that to get the point count
let byte_len = decompressor.decompress_until_end_of_file(&mut data_to_recompress)?;
data_to_recompress.resize(byte_len, 0);
// The last chunk is going to be rewritten, and will be completed later
let _ = chunk_table.pop();
}
// For variable size chunks, we don't need to re-read the last chunk since
// each chunk can have its own size
// Seek to beginning of data, so that the compressor can be properly initialized
data.seek(SeekFrom::Start(start_of_data))?;
let mut compressor = compressor_creator(data, vlr)?;
// Explicitly reserve the offset so that the compressor knows where the
// offset is.
compressor.reserve_offset_to_chunk_table()?;
// Rewrite the last chunk
let last_chunk_pos = chunk_table.chunk_position(chunk_table.len()).unwrap();
get_mut_dest_of_compressor(&mut compressor).seek(SeekFrom::Current(last_chunk_pos as i64))?;
if !data_to_recompress.is_empty() {
compressor.compress_many(&data_to_recompress)?;
}
Ok((compressor, chunk_table))
}
/// Struct that handles appending compressed points to a LAZ file.
pub struct LasZipAppender<'a, W: Write + Send + 'a> {
saved_chunk_table: ChunkTable,
compressor: LasZipCompressor<'a, W>,
}
impl<'a, W> LasZipAppender<'a, W>
where
W: Read + Write + Seek + Send + 'a,
{
/// data must be positioned at the start of point data
pub fn new(data: W, vlr: LazVlr) -> crate::Result<Self> {
let (compressor, chunk_table) = prepare_compressor_for_appending(
data,
vlr,
LasZipCompressor::new,
LasZipCompressor::get_mut,
)?;
Ok(Self {
saved_chunk_table: chunk_table,
compressor,
})
}
/// Tells the compressor that no more points will be compressed
///
/// - Compresses & writes the rest of the points to form the last chunk
/// - Writes the chunk table
/// - update the offset to the chunk_table
pub fn done(&mut self) -> crate::Result<()> {
self.compressor.done()?;
// The compressor wrote a chunk table that only corresponds to added chunks
// We have to write the chunk table that also have the original chunks
// 1. Get position of chunk table
let pos = self.compressor.chunk_table_position_offset() as u64;
self.compressor.get_mut().seek(SeekFrom::Start(pos))?;
let (_, chunk_table_pos) = ChunkTable::read_offset(self.compressor.get_mut())?
.expect("Somehow, the chunk table was not written");
self.saved_chunk_table.extend(self.compressor.chunk_table());
// 2 .Overwrite with the correct chunk table
let write_point_count = self.compressor.vlr().uses_variable_size_chunks();
let dest = self.compressor.get_mut();
dest.seek(SeekFrom::Start(chunk_table_pos))?;
self.saved_chunk_table.write(dest, write_point_count)?;
Ok(())
}
pub fn get_mut(&mut self) -> &mut W {
self.compressor.get_mut()
}
pub fn get(&self) -> &W {
self.compressor.get()
}
pub fn into_inner(self) -> W {
self.compressor.into_inner()
}
}
impl<'a, W> LasZipAppender<'a, W>
where
W: Write + Seek + Send + 'a,
{
/// Compress the point and write the compressed data to the destination given when
/// the compressor was constructed
///
/// The data is written in the buffer is expected to be exactly
/// as it would have been in a LAS File, that is:
///
/// - The fields/dimensions are in the same order as the LAS spec says
/// - The data in the buffer is in Little Endian order
pub fn compress_one(&mut self, input: &[u8]) -> std::io::Result<()> {
self.compressor.compress_one(input)
}
/// Compresses many points using multiple threads.
///
/// # Important
///
/// This **must** be called **only** when writing **fixed-size** chunks.
/// This will **panic** otherwise.
///
/// # Note
///
/// For this function to actually use multiple threads, the `points`
/// buffer shall hold more points that the vlr's `chunk_size`.
pub fn compress_many(&mut self, points: &[u8]) -> std::io::Result<()> {
self.compressor.compress_many(points)
}
/// Compresses multiple chunks
///
/// # Important
///
/// This **must** be called **only** when writing **variable-size** chunks.
/// This will **panic** otherwise.
pub fn compress_chunks<Chunks, Item>(&mut self, chunks: Chunks) -> std::io::Result<()>
where
Item: AsRef<[u8]> + Send,
Chunks: IntoIterator<Item = Item>,
{
self.compressor.compress_chunks(chunks)
}
/// Finished the current chunks.
///
/// All points compressed with the previous calls to [`compress_one`] and [`compress_many`]
/// will form one chunk. And the subsequent calls to [`compress_one`] and [`compress_many`]
/// will form a new chunk.
///
/// # Important
///
/// Only call this when writing **variable-size** chunks.
///
///
/// [`compress_one`]: Self::compress_one
/// [`compress_many`]: Self::compress_many
pub fn finish_current_chunk(&mut self) -> std::io::Result<()> {
self.compressor.finish_current_chunk()
}
}